def main(mode, workflow, problem, run, **kwargs):
"""Create & submit the DH search via Balsam"""
job = pre_submit(problem, run, workflow)
if os.path.exists(problem):
problem = os.path.abspath(problem)
if os.path.exists(run):
run = os.path.abspath(run)
if mode == "nas":
print("Creating NAS(PPO) BalsamJob...", end="", flush=True)
setup_nas(job, problem, run, **kwargs)
print("OK")
elif mode == "hps":
print("Creating HPS(AMBS) BalsamJob...", end="", flush=True)
setup_ambs(job, problem, run, **kwargs)
print("OK")
print("Performing job submission...")
submit_qlaunch(
kwargs["project"],
kwargs["queue"],
kwargs["nodes"],
kwargs["time_minutes"],
kwargs["job_mode"],
workflow,
)
banner(f"Success. The search will run at: {job.working_directory}")
def __init__(
self,
problem: str,
run: str,
evaluator: str,
max_evals: int = 1000000,
seed: int = None,
num_nodes_master: int = 1,
num_workers: int = None,
log_dir: int = None,
**kwargs,
):
kwargs["problem"] = problem
kwargs["run"] = run
kwargs["evaluator"] = evaluator
kwargs["max_evals"] = max_evals # * For retro compatibility
kwargs["seed"] = seed
# Loading problem instance and run function
self.problem = util.generic_loader(problem, "Problem")
if self.problem.seed == None:
self.problem.seed = seed
else:
kwargs["seed"] = self.problem.seed
self.run_func = util.generic_loader(run, "run")
notice = f"Maximizing the return value of function: {run}"
logger.info(notice)
util.banner(notice)
self.evaluator = Evaluator.create(
self.run_func,
method=evaluator,
num_nodes_master=num_nodes_master,
num_workers=num_workers,
**kwargs,
)
self.num_workers = self.evaluator.num_workers
self.max_evals = max_evals
self.log_dir = os.getcwd() if log_dir is None else log_dir
# set the random seed
np.random.seed(self.problem.seed)
logger.info(f"Options: " + pformat(kwargs, indent=4))
logger.info("Hyperparameter space definition: " +
pformat(self.problem.space, indent=4))
logger.info(f"Created {evaluator} evaluator")
logger.info(f"Evaluator: num_workers is {self.num_workers}")
def __init__(self,
problem: str,
run: str,
evaluator: str,
max_evals: int = 100,
seed: int = None,
**kwargs):
kwargs['problem'] = problem
kwargs['run'] = run
kwargs['evaluator'] = evaluator
kwargs['max_evals'] = max_evals # * For retro compatibility
kwargs['seed'] = seed
self.problem = util.generic_loader(problem, 'Problem')
if self.problem.seed == None:
self.problem.seed = seed
else:
kwargs['seed'] = self.problem.seed
self.run_func = util.generic_loader(run, 'run')
notice = f'Maximizing the return value of function: {run}'
logger.info(notice)
util.banner(notice)
self.evaluator = Evaluator.create(self.run_func,
method=evaluator,
**kwargs)
self.num_workers = self.evaluator.num_workers
self.max_evals = max_evals
# set the random seed
np.random.seed(self.problem.seed)
logger.info(f'Options: ' + pformat(kwargs, indent=4))
logger.info('Hyperparameter space definition: ' +
pformat(self.problem.space, indent=4))
logger.info(f'Created {evaluator} evaluator')
logger.info(f'Evaluator: num_workers is {self.num_workers}')
def main(
mode,
search,
workflow,
problem,
run,
time_minutes,
nodes,
queue,
project,
max_evals,
num_cpus_per_task,
num_gpus_per_task,
activation_script,
horovod,
**kwargs,
):
"""Create & submit the DH search via Balsam"""
activation_script = os.path.abspath(activation_script)
# Test if "run", "problem" and "workflow" are correct
validate(problem, run, workflow)
# Creation of the submission script
# Detection of the host
hostname = socket.gethostname()
# hostname = "thetagpusn1"
host = None
if "thetagpu" in hostname:
host = "thetagpu"
print("ThetaGPU detected")
else:
print(
f"There exist no submission policy for the current system: '{hostname}'"
)
exit()
# Load submission template
job_template_path = os.path.join(MODULE_PATH, "job-templates-ray",
f"{host}.submission.tmpl")
with open(job_template_path, "r") as f:
template = Template(f.read())
# Load script to launch ray cluster template
if nodes > 1: # mutliple nodes
launch_ray_path = os.path.join(MODULE_PATH, "job-templates-ray",
f"{host}.MultiNodesRayCluster.tmpl")
else: # single node
launch_ray_path = os.path.join(MODULE_PATH, "job-templates-ray",
f"{host}.SingleNodeRayCluster.tmpl")
with open(launch_ray_path, "r") as f:
template_launch_ray = Template(f.read())
# Render script to launch ray cluster
script_launch_ray_cluster = template_launch_ray.render()
# Create workflow directory and move ot it
current_dir = os.getcwd()
exp_dir = os.path.join(current_dir, workflow)
if not (os.path.exists(exp_dir)):
create_dir(exp_dir)
os.chdir(exp_dir)
# resolve the evaluator to use
if horovod:
evaluator = "rayhorovod"
else:
evaluator = "ray"
# Render submission template
submission_path = os.path.join(exp_dir, f"sub_{workflow}.sh")
with open(submission_path, "w") as fp:
fp.write(
template.render(
mode=mode,
search=search,
evaluator=evaluator,
problem=problem,
run=run,
time_minutes=time_minutes,
nodes=nodes,
queue=queue,
project=project,
max_evals=max_evals,
num_cpus_per_task=num_cpus_per_task,
num_gpus_per_task=num_gpus_per_task,
script_launch_ray_cluster=script_launch_ray_cluster,
activation_script=activation_script,
other_search_arguments=generate_other_arguments(**kwargs),
))
print("Created", fp.name)
# add executable rights
st = os.stat(submission_path)
os.chmod(submission_path, st.st_mode | stat.S_IEXEC)
# Job submission
print("Performing job submission...")
cmd = f"qsub {submission_path}"
os.system(cmd)
banner(f"Success. The search will run at: {exp_dir}")