def main(args, logger: Logger = get_basic_logger()):
params = utils.load_sim_params(os.path.join(args.rel_dir, "sim_params.yaml"))
sim_dir = params.sim_dir
mgmt_db_loc = params.mgmt_db_location
submit_yes = True if args.auto else confirm("Also submit the job for you?")
# get the srf(rup) name without extensions
srf_name = os.path.splitext(os.path.basename(params.srf_file))[0]
# if srf(variation) is provided as args, only create the slurm
# with same name provided
if args.srf is not None and srf_name != args.srf:
return
write_directory = args.write_directory if args.write_directory else sim_dir
# get lf_sim_dir
lf_sim_dir = os.path.join(sim_dir, "LF")
header_dict = {
"platform_specific_args": get_platform_node_requirements(
platform_config[const.PLATFORM_CONFIG.MERGE_TS_DEFAULT_NCORES.name]
),
"wallclock_limit": default_run_time_merge_ts,
"job_name": "merge_ts.{}".format(srf_name),
"job_description": "post emod3d: merge_ts",
"additional_lines": "###SBATCH -C avx",
}
command_template_parameters = {
"run_command": platform_config[const.PLATFORM_CONFIG.RUN_COMMAND.name],
"merge_ts_path": binary_version.get_unversioned_bin(
"merge_tsP3_par", get_machine_config(args.machine)["tools_dir"]
),
}
body_template_params = (
"{}.sl.template".format(merge_ts_name_prefix),
{"lf_sim_dir": lf_sim_dir},
)
script_prefix = "{}_{}".format(merge_ts_name_prefix, srf_name)
script_file_path = write_sl_script(
write_directory,
sim_dir,
const.ProcessType.merge_ts,
script_prefix,
header_dict,
body_template_params,
command_template_parameters,
)
if submit_yes:
submit_script_to_scheduler(
script_file_path,
const.ProcessType.merge_ts.value,
sim_struct.get_mgmt_db_queue(mgmt_db_loc),
sim_dir,
srf_name,
target_machine=args.machine,
logger=logger,
)
def submit_script_to_scheduler(script_name, target_machine=None, **kwargs):
shared_automated_workflow.submit_script_to_scheduler(
script_name,
proc_type,
sim_struct.get_mgmt_db_queue(root_folder),
sim_dir,
run_name,
target_machine=target_machine,
logger=task_logger,
)
def main(
args: argparse.Namespace,
est_model: est.EstModel = None,
logger: Logger = get_basic_logger(),
):
params = utils.load_sim_params(
os.path.join(args.rel_dir, "sim_params.yaml"))
submit_yes = True if args.auto else confirm("Also submit the job for you?")
logger.debug("params.srf_file {}".format(params.srf_file))
# Get the srf(rup) name without extensions
srf_name = os.path.splitext(os.path.basename(params.srf_file))[0]
if args.srf is None or srf_name == args.srf:
logger.debug("not set_params_only")
# get lf_sim_dir
sim_dir = os.path.abspath(params.sim_dir)
lf_sim_dir = sim_struct.get_lf_dir(sim_dir)
# default_core will be changed is user passes ncore
nt = int(float(params.sim_duration) / float(params.dt))
target_qconfig = get_machine_config(args.machine)
retries = args.retries if hasattr(args, "retries") else None
est_cores, est_run_time, wct = get_lf_cores_and_wct(
est_model,
logger,
nt,
params,
sim_dir,
srf_name,
target_qconfig,
args.ncore,
retries,
)
binary_path = binary_version.get_lf_bin(params.emod3d.emod3d_version,
target_qconfig["tools_dir"])
# use the original estimated run time for determining the checkpoint, or uses a minimum of 3 checkpoints
steps_per_checkpoint = int(
min(nt / (60.0 * est_run_time) * const.CHECKPOINT_DURATION,
nt // 3))
write_directory = (args.write_directory
if args.write_directory else params.sim_dir)
set_runparams.create_run_params(
sim_dir, steps_per_checkpoint=steps_per_checkpoint, logger=logger)
header_dict = {
"wallclock_limit": wct,
"job_name": "emod3d.{}".format(srf_name),
"job_description": "emod3d slurm script",
"additional_lines": "#SBATCH --hint=nomultithread",
"platform_specific_args":
get_platform_node_requirements(est_cores),
}
command_template_parameters = {
"run_command":
platform_config[const.PLATFORM_CONFIG.RUN_COMMAND.name],
"emod3d_bin": binary_path,
"lf_sim_dir": lf_sim_dir,
}
body_template_params = ("run_emod3d.sl.template", {})
script_prefix = "run_emod3d_{}".format(srf_name)
script_file_path = write_sl_script(
write_directory,
params.sim_dir,
const.ProcessType.EMOD3D,
script_prefix,
header_dict,
body_template_params,
command_template_parameters,
)
if submit_yes:
submit_script_to_scheduler(
script_file_path,
const.ProcessType.EMOD3D.value,
sim_struct.get_mgmt_db_queue(params.mgmt_db_location),
params.sim_dir,
srf_name,
target_machine=args.machine,
logger=logger,
)
def queue_monitor_loop(
root_folder: str,
sleep_time: int,
max_retries: int,
queue_logger: Logger = qclogging.get_basic_logger(),
alert_url=None,
):
mgmt_db = MgmtDB(sim_struct.get_mgmt_db(root_folder))
queue_folder = sim_struct.get_mgmt_db_queue(root_folder)
queue_logger.info("Running queue-monitor, exit with Ctrl-C.")
mgmt_db.add_retries(max_retries)
sqlite_tmpdir = "/tmp/cer"
while keepAlive:
complete_data = True
if not os.path.exists(sqlite_tmpdir):
os.makedirs(sqlite_tmpdir)
queue_logger.debug("Set up the sqlite_tmpdir")
# For each hpc get a list of job id and status', and for each pair save them in a dictionary
queued_tasks = {}
for hpc in HPC:
try:
squeued_tasks = Scheduler.get_scheduler().check_queues(
user=False, target_machine=hpc
)
except EnvironmentError as e:
queue_logger.critical(e)
queue_logger.critical(
f"An error was encountered when attempting to check {Scheduler.get_scheduler().QUEUE_NAME} for HPC {hpc}. "
"Tasks will not be submitted to this HPC until the issue is resolved"
)
complete_data = False
else:
for task in squeued_tasks:
queued_tasks[task.split()[0]] = task.split()[1]
if len(queued_tasks) > 0:
if len(queued_tasks) > 200:
queue_logger.log(
VERYVERBOSE,
f"{Scheduler.get_scheduler().QUEUE_NAME} tasks: {', '.join([' '.join(task) for task in queued_tasks.items()])}",
)
queue_logger.info(
f"Over 200 tasks were found in the queue. Check the log for an exact listing of them"
)
else:
queue_logger.info(
f"{Scheduler.get_scheduler().QUEUE_NAME} tasks: {', '.join([' '.join(task) for task in queued_tasks.items()])}"
)
else:
queue_logger.debug(f"No {Scheduler.get_scheduler().QUEUE_NAME} tasks")
db_in_progress_tasks = mgmt_db.get_submitted_tasks()
if len(db_in_progress_tasks) > 0:
queue_logger.info(
"In progress tasks in mgmt db:"
+ ", ".join(
[
"{}-{}-{}-{}".format(
entry.run_name,
const.ProcessType(entry.proc_type).str_value,
entry.job_id,
const.Status(entry.status).str_value,
)
for entry in db_in_progress_tasks
]
)
)
entry_files = os.listdir(queue_folder)
entry_files.sort()
entries = []
for file_name in entry_files[::-1]:
queue_logger.debug(
"Checking {} to see if it is a valid update file".format(file_name)
)
entry = get_queue_entry(os.path.join(queue_folder, file_name), queue_logger)
if entry is None:
queue_logger.debug(
"Removing {} from the list of update files".format(file_name)
)
entry_files.remove(file_name)
else:
if str(entry.job_id) in queued_tasks.keys() and entry.status > 3:
# This will prevent race conditions if the failure/completion state file is made and picked up before the job actually finishes
# Most notabley happens on Kisti
# The queued and running states are allowed
queue_logger.debug(
"Job {} is still running on the HPC, skipping this iteration".format(
entry
)
)
entry_files.remove(file_name)
else:
queue_logger.debug("Adding {} to the list of updates".format(entry))
entries.insert(0, entry)
entries.extend(
update_tasks(
entry_files,
queued_tasks,
db_in_progress_tasks,
complete_data,
queue_logger,
root_folder,
)
)
if len(entries) > 0:
queue_logger.info("Updating {} mgmt db tasks.".format(len(entries)))
if mgmt_db.update_entries_live(entries, max_retries, queue_logger):
for file_name in entry_files:
os.remove(os.path.join(queue_folder, file_name))
# check for jobs that matches alert criteria
if alert_url != None:
for entry in entries:
if entry.status == const.Status.failed.value:
entry_retries = mgmt_db.get_retries(
entry.proc_type, entry.run_name
)
if entry_retries < max_retries:
msg = f"fault:{entry.run_name} step:{entry.proc_type} has failed with error:{entry.error}"
elif entry_retries >= max_retries:
msg = f"@here fault:{entry.run_name} step:{entry.proc_type} has failed with error:{entry.error} and met the retry cap"
send_alert(msg, alert_url)
else:
queue_logger.error(
"Failed to update the current entries in the mgmt db queue. "
"Please investigate and fix. If this is a repeating error, then this "
"will block all other entries from updating."
)
else:
queue_logger.info("No entries in the mgmt db queue.")
# Nap time
queue_logger.debug("Sleeping for {}".format(sleep_time))
time.sleep(sleep_time)
def main(
args: argparse.Namespace,
est_model: est.EstModel = None,
logger: Logger = get_basic_logger(),
):
params = utils.load_sim_params(
os.path.join(args.rel_dir, "sim_params.yaml"))
# check if the args is none, if not, change the version
if args.version is not None and args.version in ["mpi", "run_hf_mpi"]:
version = args.version
ll_name_prefix = "run_hf_mpi"
else:
if args.version is not None:
logger.error(
"{} cannot be recognize as a valid version option. version is set to default: {}"
.format(
args.version,
platform_config[
const.PLATFORM_CONFIG.HF_DEFAULT_VERSION.name],
))
version = platform_config[
const.PLATFORM_CONFIG.HF_DEFAULT_VERSION.name]
ll_name_prefix = platform_config[
const.PLATFORM_CONFIG.HF_DEFAULT_VERSION.name]
logger.debug("version: {}".format(version))
# modify the logic to use the same as in install_bb:
# sniff through params_base to get the names of srf,
# instead of running through file directories.
# loop through all srf file to generate related slurm scripts
srf_name = os.path.splitext(os.path.basename(params.srf_file))[0]
# if srf(variation) is provided as args, only create
# the slurm with same name provided
if args.srf is None or srf_name == args.srf:
nt = get_hf_nt(params)
fd_count = len(shared.get_stations(params.FD_STATLIST))
# TODO:make it read through the whole list
# instead of assuming every stoch has same size
nsub_stoch, sub_fault_area = srf.get_nsub_stoch(params.hf.slip,
get_area=True)
if est_model is None:
est_model = os.path.join(
platform_config[
const.PLATFORM_CONFIG.ESTIMATION_MODELS_DIR.name], "HF")
est_core_hours, est_run_time, est_cores = est.est_HF_chours_single(
fd_count,
nsub_stoch,
nt,
args.ncore,
est_model,
scale_ncores=SCALE_NCORES,
logger=logger,
)
# scale up the est_run_time if it is a re-run (with check-pointing)
# creates and extra variable so we keep the orignial estimated run time for other purpose
est_run_time_scaled = est_run_time
if hasattr(args, "retries") and int(args.retries) > 0:
# check if HF.bin is read-able = restart-able
try:
from qcore.timeseries import HFSeis
bin = HFSeis(sim_struct.get_hf_bin_path(params.sim_dir))
except:
logger.debug("Retried count > 0 but HF.bin is not readable")
else:
est_run_time_scaled = est_run_time * (int(args.retries) + 1)
wct = set_wct(est_run_time_scaled, est_cores, args.auto)
hf_sim_dir = sim_struct.get_hf_dir(params.sim_dir)
write_directory = (args.write_directory
if args.write_directory else params.sim_dir)
underscored_srf = srf_name.replace("/", "__")
header_dict = {
"platform_specific_args":
get_platform_node_requirements(est_cores),
"wallclock_limit": wct,
"job_name": "hf.{}".format(underscored_srf),
"job_description": "HF calculation",
"additional_lines": "###SBATCH -C avx",
}
command_template_parameters, add_args = gen_command_template(
params, args.machine, seed=args.seed)
body_template_params = (
"{}.sl.template".format(ll_name_prefix),
{
"hf_sim_dir": hf_sim_dir,
"test_hf_script": "test_hf.sh"
},
)
script_prefix = "{}_{}".format(ll_name_prefix, underscored_srf)
script_file_path = write_sl_script(
write_directory,
params.sim_dir,
const.ProcessType.HF,
script_prefix,
header_dict,
body_template_params,
command_template_parameters,
add_args,
)
# Submit the script
submit_yes = True if args.auto else confirm(
"Also submit the job for you?")
if submit_yes:
submit_script_to_scheduler(
script_file_path,
const.ProcessType.HF.value,
sim_struct.get_mgmt_db_queue(params.mgmt_db_location),
params.sim_dir,
srf_name,
target_machine=args.machine,
logger=logger,
)
def main(
args: argparse.Namespace,
est_model: est.EstModel = None,
logger: Logger = get_basic_logger(),
):
params = utils.load_sim_params(os.path.join(args.rel_dir, "sim_params.yaml"))
ncores = platform_config[const.PLATFORM_CONFIG.BB_DEFAULT_NCORES.name]
version = args.version
if version in ["mpi", "run_bb_mpi"]:
sl_name_prefix = "run_bb_mpi"
else:
if version is not None:
logger.error(
"{} cannot be recognized as a valid option. version is set to default:".format(
version,
platform_config[const.PLATFORM_CONFIG.BB_DEFAULT_VERSION.name],
)
)
version = platform_config[const.PLATFORM_CONFIG.BB_DEFAULT_VERSION.name]
sl_name_prefix = platform_config[const.PLATFORM_CONFIG.BB_DEFAULT_VERSION.name]
logger.debug(version)
srf_name = os.path.splitext(os.path.basename(params.srf_file))[0]
if args.srf is None or srf_name == args.srf:
# TODO: save status as HF. refer to submit_hf
# Use HF nt for wct estimation
nt = get_hf_nt(params)
fd_count = len(shared.get_stations(params.FD_STATLIST))
if est_model is None:
est_model = os.path.join(
platform_config[const.PLATFORM_CONFIG.ESTIMATION_MODELS_DIR.name], "BB"
)
est_core_hours, est_run_time = est.est_BB_chours_single(
fd_count, nt, ncores, est_model
)
# creates and extra variable so we keep the original estimated run time for other purpos
est_run_time_scaled = est_run_time
if hasattr(args, "retries") and int(args.retries) > 0:
# check if BB.bin is read-able = restart-able
try:
from qcore.timeseries import BBSeis
bin = BBSeis(simulation_structure.get_bb_bin_path(params.sim_dir))
except:
logger.debug("Retried count > 0 but BB.bin is not readable")
else:
est_run_time_scaled = est_run_time * (int(args.retries) + 1)
wct = set_wct(est_run_time_scaled, ncores, args.auto)
write_directory = (
args.write_directory if args.write_directory else params.sim_dir
)
underscored_srf = srf_name.replace("/", "__")
header_dict = {
"wallclock_limit": wct,
"job_name": "bb.{}".format(underscored_srf),
"job_description": "BB calculation",
"additional_lines": "###SBATCH -C avx",
"platform_specific_args": get_platform_node_requirements(ncores),
}
body_template_params = (
"{}.sl.template".format(sl_name_prefix),
{"test_bb_script": "test_bb.sh"},
)
command_template_parameters, add_args = gen_command_template(params)
script_prefix = "{}_{}".format(sl_name_prefix, underscored_srf)
script_file_path = write_sl_script(
write_directory,
params.sim_dir,
const.ProcessType.BB,
script_prefix,
header_dict,
body_template_params,
command_template_parameters,
add_args,
)
# Submit the script
submit_yes = True if args.auto else confirm("Also submit the job for you?")
if submit_yes:
submit_script_to_scheduler(
script_file_path,
const.ProcessType.BB.value,
simulation_structure.get_mgmt_db_queue(params.mgmt_db_location),
params.sim_dir,
srf_name,
target_machine=args.machine,
logger=logger,
)
def submit_im_calc_slurm(
sim_dir: str,
write_dir: str = None,
simple_out: bool = True,
adv_ims: bool = False,
target_machine: str = get_target_machine(
const.ProcessType.IM_calculation).name,
est_model: EstModel = path.join(
platform_config[const.PLATFORM_CONFIG.ESTIMATION_MODELS_DIR.name],
"IM"),
logger: Logger = get_basic_logger(),
):
"""Creates the IM calc slurm scrip, also submits if specified
The options_dict is populated by the DEFAULT_OPTIONS, values can be changed by
passing in a dict containing the entries that require changing. Merges the
two dictionaries, the passed in one has higher priority.
"""
# Load the yaml params
params = utils.load_sim_params(
sim_struct.get_sim_params_yaml_path(sim_dir), load_vm=True)
realisation_name = params[const.SimParams.run_name.value]
fault_name = sim_struct.get_fault_from_realisation(realisation_name)
station_count = len(load_station_file(params["FD_STATLIST"]).index)
header_options = {
const.SlHdrOptConsts.description.value:
"Calculates intensity measures.",
const.SlHdrOptConsts.memory.value:
"2G",
const.SlHdrOptConsts.version.value:
"slurm",
"exe_time":
const.timestamp,
const.SlHdrOptConsts.additional.value:
"#SBATCH --hint=nomultithread"
if platform_config[const.PLATFORM_CONFIG.SCHEDULER.name] == "slurm"
else [""],
}
body_options = {
const.SlBodyOptConsts.component.value: "",
"realisation_name": realisation_name,
const.SlBodyOptConsts.fault_name.value: fault_name,
"np":
platform_config[const.PLATFORM_CONFIG.IM_CALC_DEFAULT_N_CORES.name],
"sim_IM_calc_dir": sim_struct.get_im_calc_dir(sim_dir),
"output_csv": sim_struct.get_IM_csv(sim_dir),
"output_info": sim_struct.get_IM_info(sim_dir),
"models": "",
const.SlBodyOptConsts.mgmt_db.value: "",
"n_components": "",
"match_obs_stations": False,
}
command_options = {
const.SlBodyOptConsts.sim_dir.value:
sim_dir,
const.SlBodyOptConsts.component.value:
"",
const.SlBodyOptConsts.sim_name.value:
realisation_name,
const.SlBodyOptConsts.fault_name.value:
fault_name,
const.SlBodyOptConsts.n_procs.value:
platform_config[const.PLATFORM_CONFIG.IM_CALC_DEFAULT_N_CORES.name],
const.SlBodyOptConsts.extended.value:
"",
const.SlBodyOptConsts.simple_out.value:
"",
const.SlBodyOptConsts.advanced_IM.value:
"",
"pSA_periods":
"",
}
# Convert option settings to values
if write_dir is None:
write_dir = sim_dir
# Simple vs adv im settings
if adv_ims:
# Common values
proc_type = const.ProcessType.advanced_IM
sl_template = "adv_im_calc.sl.template"
script_prefix = "adv_im_calc"
body_options["models"] = " ".join(
params[const.SlBodyOptConsts.advanced_IM.value]["models"])
command_options[
const.SlBodyOptConsts.advanced_IM.
value] = f"-a {body_options['models']} --OpenSees {qconfig['OpenSees']} "
# create temporary station list if "match_obs_stations" is directory
if path.isdir(
str(params[const.SlBodyOptConsts.advanced_IM.value]
["match_obs_stations"])):
logger.debug(
"match_obs_station specificed: {params[const.SlBodyOptConsts.advanced_IM.value]['match_obs_stations']}"
)
# retreived station list from observed/fault(eventname)/Vol*/data/accBB/station.
obs_accBB_dir_glob = path.join(
params[const.SlBodyOptConsts.advanced_IM.value]
["match_obs_stations"],
f"{fault_name}/*/*/accBB",
)
obs_accBB_dir = glob.glob(obs_accBB_dir_glob)
if len(obs_accBB_dir) > 1:
logger.error(
"got more than one folder globbed. please double check the path to the match_obs_stations is correct."
)
sys.exit()
station_names_tmp = get_observed_stations(obs_accBB_dir[0])
# write to a tmp file
tmp_station_file = path.join(sim_dir, "tmp_station_file")
with open(tmp_station_file, "w") as f:
for station in station_names_tmp:
f.write(f"{station} ")
command_options[const.SlBodyOptConsts.advanced_IM.value] = (
command_options[const.SlBodyOptConsts.advanced_IM.value] +
f"--station_names `cat {tmp_station_file}`")
# header_options[const.SlHdrOptConsts.n_tasks.value] = body_options["np"] = qconfig["cores_per_node"]
# Time for one station to run in hours
# This should be a machine property. Or take the largest across all machines used
time_for_one_station = 0.5
est_run_time = (np.ceil(station_count / qconfig["cores_per_node"]) *
2 * time_for_one_station)
else:
proc_type = const.ProcessType.IM_calculation
sl_template = "sim_im_calc.sl.template"
script_prefix = "sim_im_calc"
if simple_out:
command_options[const.SlBodyOptConsts.simple_out.value] = "-s"
if params["ims"][const.RootParams.extended_period.name]:
command_options[const.SlBodyOptConsts.extended.value] = "-e"
period_count = len(
np.unique(
np.append(params["ims"]["pSA_periods"], const.EXT_PERIOD)))
else:
period_count = len(params["ims"]["pSA_periods"])
if "pSA_periods" in params["ims"]:
command_options[
"pSA_periods"] = f"-p {' '.join(str(p) for p in params['ims']['pSA_periods'])}"
comps_to_store = params["ims"][const.SlBodyOptConsts.component.value]
command_options[const.SlBodyOptConsts.component.
value] = "-c " + " ".join(comps_to_store)
body_options["n_components"] = len(comps_to_store)
# Get wall clock estimation
logger.info(
"Running wall clock estimation for IM sim for realisation {}".
format(realisation_name))
_, est_run_time = est_IM_chours_single(
station_count,
int(float(params["sim_duration"]) / float(params["dt"])),
comps_to_store,
period_count,
body_options["np"],
est_model,
)
# Header options requiring upstream settings
# special treatment for im_calc, as the scaling feature in estimation is not suitable
# cap the wct, otherwise cannot submit
est_run_time = min(est_run_time * CH_SAFETY_FACTOR, qconfig["MAX_JOB_WCT"])
# set ch_safety_factor=1 as we scale it already.
header_options["wallclock_limit"] = get_wct(est_run_time,
ch_safety_factor=1)
logger.debug("Using WCT for IM_calc: {header_options['wallclock_limit']}")
header_options["job_name"] = "{}_{}".format(proc_type.str_value,
fault_name)
header_options["platform_specific_args"] = get_platform_node_requirements(
body_options["np"])
script_file_path = write_sl_script(
write_dir,
sim_dir,
proc_type,
script_prefix,
header_options,
(sl_template, body_options),
command_options,
)
submit_script_to_scheduler(
script_file_path,
proc_type.value,
sim_struct.get_mgmt_db_queue(params["mgmt_db_location"]),
sim_dir,
realisation_name,
target_machine=target_machine,
logger=logger,
)
def run_main_submit_loop(
root_folder: str,
n_runs: Dict[str, int],
rels_to_run: str,
given_tasks_to_run: List[const.ProcessType],
sleep_time: int,
models_tuple: Tuple[est.EstModel],
main_logger: Logger = qclogging.get_basic_logger(),
cycle_timeout=1,
):
mgmt_queue_folder = sim_struct.get_mgmt_db_queue(root_folder)
mgmt_db = MgmtDB(sim_struct.get_mgmt_db(root_folder))
root_params_file = os.path.join(sim_struct.get_runs_dir(root_folder),
"root_params.yaml")
config = utils.load_yaml(root_params_file)
main_logger.info("Loaded root params file: {}".format(root_params_file))
# Default values
hf_seed = config["hf"].get(const.RootParams.seed.value,
const.HF_DEFAULT_SEED)
main_logger.debug("hf_seed set to {}".format(hf_seed))
main_logger.debug(
f"extended_period set to {config['ims']['extended_period']}")
time_since_something_happened = cycle_timeout
while time_since_something_happened > 0:
main_logger.debug("time_since_something_happened is now {}".format(
time_since_something_happened))
time_since_something_happened -= 1
# Get items in the mgmt queue, have to get a snapshot instead of
# checking the directory real-time to prevent timing issues,
# which can result in dual-submission
mgmt_queue_entries = os.listdir(mgmt_queue_folder)
# Get in progress tasks in the db and the HPC queue
n_tasks_to_run = {}
for hpc in HPC:
try:
squeued_tasks = Scheduler.get_scheduler().check_queues(
user=True, target_machine=hpc)
except EnvironmentError as e:
main_logger.critical(e)
n_tasks_to_run[hpc] = 0
else:
n_tasks_to_run[hpc] = n_runs[hpc] - len(squeued_tasks)
if len(squeued_tasks) > 0:
main_logger.debug(
"There was at least one job in squeue, resetting timeout"
)
time_since_something_happened = cycle_timeout
# Gets all runnable tasks based on mgmt db state
runnable_tasks = mgmt_db.get_runnable_tasks(
rels_to_run,
sum(n_runs.values()),
os.listdir(sim_struct.get_mgmt_db_queue(root_folder)),
given_tasks_to_run,
main_logger,
)
if len(runnable_tasks) > 0:
time_since_something_happened = cycle_timeout
main_logger.info("Number of runnable tasks: {}".format(
len(runnable_tasks)))
main_logger.debug(
"There was at least one runnable task, resetting timeout")
else:
main_logger.debug("No runnable_tasks")
# Select the first ntask_to_run that are not waiting
# for mgmt db updates (i.e. items in the queue)
tasks_to_run, task_counter = [], {key: 0 for key in HPC}
for cur_proc_type, cur_run_name, retries in runnable_tasks:
cur_hpc = get_target_machine(cur_proc_type)
# Add task if limit has not been reached and there are no
# outstanding mgmt db updates
if (not shared_automated_workflow.check_mgmt_queue(
mgmt_queue_entries, cur_run_name, cur_proc_type) and
task_counter.get(cur_hpc, 0) < n_tasks_to_run[cur_hpc]):
tasks_to_run.append((cur_proc_type, cur_run_name, retries))
task_counter[cur_hpc] += 1
# Open to better suggestions
# Break if enough tasks for each HPC have been added
if np.all([
True if task_counter.get(hpc, 0) >= n_tasks_to_run[hpc]
else False for hpc in n_tasks_to_run.keys()
]):
break
if len(tasks_to_run) > 0:
main_logger.info("Tasks to run this iteration: " + ", ".join([
"{}-{}".format(entry[1],
const.ProcessType(entry[0]).str_value)
for entry in tasks_to_run
]))
else:
main_logger.debug("No tasks to run this iteration")
# Submit the runnable tasks
for proc_type, run_name, retries in tasks_to_run:
# Special handling for merge-ts
if proc_type == const.ProcessType.merge_ts.value:
# Check if clean up has already run
if mgmt_db.is_task_complete([
const.ProcessType.clean_up.value,
run_name,
const.Status.completed.str_value,
]):
# If clean_up has already run, then we should set it to
# be run again after merge_ts has run
shared_automated_workflow.add_to_queue(
mgmt_queue_folder,
run_name,
const.ProcessType.clean_up.value,
const.Status.created.value,
logger=main_logger,
)
# submit the job
submit_task(
sim_struct.get_sim_dir(root_folder, run_name),
proc_type,
run_name,
root_folder,
main_logger,
retries=retries,
hf_seed=hf_seed,
models=models_tuple,
)
main_logger.debug("Sleeping for {} second(s)".format(sleep_time))
time.sleep(sleep_time)
main_logger.info(
"Nothing was running or ready to run last cycle, exiting now")