def dictToExperiment(cls, experiment_dict):
"""Returns dict as Experiment
Args:
experiment_dict(dict): dictionary representation of Experiment
Returns:
experiment(Experiment): constructed Experiment
"""
experiment_params = [Parameter(**param) for param in experiment_dict.pop('parameters')]
if in_production:
compute = EC2Compute(**experiment_dict.pop('compute'))
else:
compute = LocalCompute(**experiment_dict.pop('compute'))
return Experiment(parameters=experiment_params, compute=compute, **experiment_dict)
def dictToExperiment(experiment_dict):
"""Returns dict as Experiment
Args:
experiment_dict(dict): dictionary representation of Experiment
Returns:
experiment(Experiment): constructed Experiment
"""
experiment_params = [
Parameter(**param) for param in experiment_dict.pop('parameters')
]
compute_type = experiment_dict['compute']['type']
if compute_type == 'ec2':
compute = EC2Compute(**experiment_dict.pop('compute'))
elif compute_type == 'local':
compute = LocalCompute(**experiment_dict.pop('compute'))
elif compute_type == 'PBSPro':
compute = PBSProCompute(**experiment_dict.pop('compute'))
return Experiment(parameters=experiment_params,
compute=compute,
**experiment_dict)
#! /bin/bash
sleep ${myParam}
sleep ${myParamB}
sleep ${myParamC}
"""
experiment_inst = Experiment(
tool_name='anothertoolaaa',
parameters=[
Parameter(name="myParam",
type=PARAMETER_TYPE_INT,
minimum=5,
maximum=10),
Parameter(name="myParamB",
type=PARAMETER_TYPE_INT,
minimum=3,
maximum=5),
Parameter(name="myParamC",
type=PARAMETER_TYPE_INT,
minimum=3,
maximum=5)
],
command_template_string=command_template_string,
# we use LocalCompute here b/c we don't want to launch jobs on EC2 like the server does
compute=LocalCompute(max_threads=8))
# when run on the server, this doesn't change - we always connect to an AWS RDS postgres database
# When running locally you can just use a sqlite database like below. The last argument is the database name
# so you could test a blank slate by just changing the name or deleting the old liteTest.db file.
storage = RelationalDB(
'sqlite',
scheduler_options=
'#PBS -P 11001079\n#PBS -l select=1:mem=1G\n#PBS -N PBSPro_paraopt_test',
worker_init='module load openmpi\nsource activate paropt')
# run on AWS (need a server to run paraopt_service, and submit experiment via paropat_sdk)
AWS_compute = EC2Compute(instance_family='c5',
instance_model='c5.2xlarge',
ami='XXXXXXX')
# run locally
LOCAL_compute = LocalCompute(max_threads=8)
# define experiment entity
experiment_inst = Experiment(tool_name='test_script',
parameters=parameters,
command_template_string=command_template_string,
setup_template_string=setup_template_string,
finish_template_string=finish_template_string,
compute=LOCAL_compute)
# define the storage
# when run on the server, this doesn't change - we always connect to an AWS RDS postgres database
# When running locally you can just use a sqlite database like below. The last argument is the database name
# so you could test a blank slate by just changing the name or deleting the old liteTest.db file.
LOCAL_storage = RelationalDB(
'sqlite',
'',
'',
'',
'liteTest',
)
def setupAWS():
# launch a small parsl job on AWS to initialize parsl's AWS VPC stuff
# If run successfully, it will create the awsproviderstate.json file on host in paropt-service/config/
# Needs to be run each time the AWS credentials are changed for the server
# Intended to be used with a `docker run ...` command before running production server
import os
import paropt
from paropt.runner import ParslRunner
from paropt.storage import RelationalDB
from paropt.optimizer import BayesianOptimizer, GridSearch
from paropt.runner.parsl import timeCommand
from paropt.storage.entities import Parameter, PARAMETER_TYPE_INT, Experiment, LocalCompute, EC2Compute
container_state_file_dir = os.getenv("CONTAINER_STATE_FILE_DIR")
if not container_state_file_dir:
raise Exception(
"Missing required env var CONTAINER_STATE_FILE_DIR which is used for copying awsproviderstate.json to host"
)
paropt.setConsoleLogger()
command_template_string = """
#! /bin/bash
sleep ${myParam}
"""
experiment_inst = Experiment(
tool_name='tmptool',
parameters=[
Parameter(name="myParam",
type=PARAMETER_TYPE_INT,
minimum=0,
maximum=10),
],
command_template_string=command_template_string,
compute=EC2Compute(
type='ec2',
instance_model=
"c4.large", # using c5 b/c previously had trouble with t2 spot instances
instance_family="c4",
ami=
"ami-0257427d05c8c18ac" # parsl base ami - preinstalled apt packages
))
# use an ephemeral database
storage = RelationalDB(
'sqlite',
'',
'',
'',
'tmpSqliteDB',
)
# run simple bayes opt
bayesian_optimizer = BayesianOptimizer(
n_init=1,
n_iter=1,
)
po = ParslRunner(parsl_app=timeCommand,
optimizer=bayesian_optimizer,
storage=storage,
experiment=experiment_inst,
logs_root_dir='/var/log/paropt')
po.run(debug=True)
po.cleanup()
# print result
print(po.run_result)
# move the awsproviderstate file into expected directory
from shutil import copyfile
copyfile("awsproviderstate.json",
f'{container_state_file_dir}/awsproviderstate.json')