def _startRunner(cls, experiment_dict, optimizer):
"""Runs an experiment with paropt. This is the function used for job queueing
Args:
experiment_dict(dict): dict representation of experiment to run.
Although it's a dict, the experiment it represents should already exist in the database.
optimizer(Optimizer): Optimizer instance to use for running the experiment
Returns:
result(dict): result of the run
Raises:
Exception: when the runner fails, it will raise an exception with the message from the result
"""
paropt.setConsoleLogger()
experiment = cls.dictToExperiment(experiment_dict)
storage = RelationalDB('postgresql', DB_USER, DB_PASSWORD, DB_HOST,
DB_NAME)
po = ParslRunner(parsl_app=timeCommand,
optimizer=optimizer,
storage=storage,
experiment=experiment,
logs_root_dir='/var/log/paropt')
po.run(debug=True)
# cleanup launched instances
po.cleanup()
if po.run_result['success'] == False:
raise Exception(po.run_result['message'])
return po.run_result
def startRunner(db_storage, experiment_dict, optimizer, obj_config):
"""Runs an experiment with paropt. This is the function used for job queueing
Args:
experiment_dict(dict): dict representation of experiment to run.
Although it's a dict, the experiment it represents should already exist in the database.
optimizer(Optimizer): Optimizer instance to use for running the experiment
Returns:
result(dict): result of the run
Raises:
Exception: when the runner fails, it will raise an exception with the message from the result
"""
paropt.setConsoleLogger()
experiment = dictToExperiment(experiment_dict)
storage = db_storage
if not os.path.exists(LOGS_ROOT_DIR):
os.makedirs(LOGS_ROOT_DIR)
po = ParslRunner(obj_func=getattr(paropt.runner.parsl,
obj_config['obj_name']),
optimizer=optimizer,
obj_func_params=obj_config['obj_params'],
storage=storage,
experiment=experiment,
logs_root_dir=LOGS_ROOT_DIR)
po.run(debug=True)
# cleanup launched instances
po.cleanup()
# if po.run_result['success'] == False:
# raise Exception(po.run_result['message'])
return po.run_result
import os
import paropt
from paropt.runner import ParslRunner
from paropt.storage import RelationalDB
from paropt.optimizer import BayesianOptimizer, GridSearch, RandomSearch
from paropt.runner.parsl import timeCmd, searchMatrix, variantCallerAccu
from paropt.storage.entities import Parameter, PARAMETER_TYPE_INT, PARAMETER_TYPE_FLOAT, Experiment, LocalCompute, EC2Compute, PBSProCompute
import json
import sys
paropt.setConsoleLogger()
# when running on server, the experiment is fetched first before doing anything
# if the experiment isn't found then running the trial fails
command_template_string = """
#! /bin/bash
sleep ${myParam}
sleep ${myParamB}
sleep ${myParamC}
"""
experiment_inst = Experiment(
tool_name='anothertoolaaa',
parameters=[
Parameter(name="myParam",
type=PARAMETER_TYPE_INT,
minimum=5,
maximum=10),
Parameter(name="myParamB",
type=PARAMETER_TYPE_INT,
def setupAWS():
# launch a small parsl job on AWS to initialize parsl's AWS VPC stuff
# If run successfully, it will create the awsproviderstate.json file on host in paropt-service/config/
# Needs to be run each time the AWS credentials are changed for the server
# Intended to be used with a `docker run ...` command before running production server
import os
import paropt
from paropt.runner import ParslRunner
from paropt.storage import RelationalDB
from paropt.optimizer import BayesianOptimizer, GridSearch
from paropt.runner.parsl import timeCommand
from paropt.storage.entities import Parameter, PARAMETER_TYPE_INT, Experiment, LocalCompute, EC2Compute
container_state_file_dir = os.getenv("CONTAINER_STATE_FILE_DIR")
if not container_state_file_dir:
raise Exception(
"Missing required env var CONTAINER_STATE_FILE_DIR which is used for copying awsproviderstate.json to host"
)
paropt.setConsoleLogger()
command_template_string = """
#! /bin/bash
sleep ${myParam}
"""
experiment_inst = Experiment(
tool_name='tmptool',
parameters=[
Parameter(name="myParam",
type=PARAMETER_TYPE_INT,
minimum=0,
maximum=10),
],
command_template_string=command_template_string,
compute=EC2Compute(
type='ec2',
instance_model=
"c4.large", # using c5 b/c previously had trouble with t2 spot instances
instance_family="c4",
ami=
"ami-0257427d05c8c18ac" # parsl base ami - preinstalled apt packages
))
# use an ephemeral database
storage = RelationalDB(
'sqlite',
'',
'',
'',
'tmpSqliteDB',
)
# run simple bayes opt
bayesian_optimizer = BayesianOptimizer(
n_init=1,
n_iter=1,
)
po = ParslRunner(parsl_app=timeCommand,
optimizer=bayesian_optimizer,
storage=storage,
experiment=experiment_inst,
logs_root_dir='/var/log/paropt')
po.run(debug=True)
po.cleanup()
# print result
print(po.run_result)
# move the awsproviderstate file into expected directory
from shutil import copyfile
copyfile("awsproviderstate.json",
f'{container_state_file_dir}/awsproviderstate.json')