def _startRunner(cls, experiment_dict, optimizer):
"""Runs an experiment with paropt. This is the function used for job queueing
Args:
experiment_dict(dict): dict representation of experiment to run.
Although it's a dict, the experiment it represents should already exist in the database.
optimizer(Optimizer): Optimizer instance to use for running the experiment
Returns:
result(dict): result of the run
Raises:
Exception: when the runner fails, it will raise an exception with the message from the result
"""
paropt.setConsoleLogger()
experiment = cls.dictToExperiment(experiment_dict)
storage = RelationalDB('postgresql', DB_USER, DB_PASSWORD, DB_HOST,
DB_NAME)
po = ParslRunner(parsl_app=timeCommand,
optimizer=optimizer,
storage=storage,
experiment=experiment,
logs_root_dir='/var/log/paropt')
po.run(debug=True)
# cleanup launched instances
po.cleanup()
if po.run_result['success'] == False:
raise Exception(po.run_result['message'])
return po.run_result
def startRunner(db_storage, experiment_dict, optimizer, obj_config):
"""Runs an experiment with paropt. This is the function used for job queueing
Args:
experiment_dict(dict): dict representation of experiment to run.
Although it's a dict, the experiment it represents should already exist in the database.
optimizer(Optimizer): Optimizer instance to use for running the experiment
Returns:
result(dict): result of the run
Raises:
Exception: when the runner fails, it will raise an exception with the message from the result
"""
paropt.setConsoleLogger()
experiment = dictToExperiment(experiment_dict)
storage = db_storage
if not os.path.exists(LOGS_ROOT_DIR):
os.makedirs(LOGS_ROOT_DIR)
po = ParslRunner(obj_func=getattr(paropt.runner.parsl,
obj_config['obj_name']),
optimizer=optimizer,
obj_func_params=obj_config['obj_params'],
storage=storage,
experiment=experiment,
logs_root_dir=LOGS_ROOT_DIR)
po.run(debug=True)
# cleanup launched instances
po.cleanup()
# if po.run_result['success'] == False:
# raise Exception(po.run_result['message'])
return po.run_result
command_template_string=command_template_string,
# we use LocalCompute here b/c we don't want to launch jobs on EC2 like the server does
compute=LocalCompute(max_threads=8))
# when run on the server, this doesn't change - we always connect to an AWS RDS postgres database
# When running locally you can just use a sqlite database like below. The last argument is the database name
# so you could test a blank slate by just changing the name or deleting the old liteTest.db file.
storage = RelationalDB(
'sqlite',
'',
'',
'',
'liteTest',
)
# when run on server, this is determined by optimizer the user POSTs
optimizer = BayesianOptimizer(n_init=2, n_iter=1, alpha=1e-3)
li = [2, 2, 2]
#optimizer = GridSearch(li)
#optimizer = RandomSearch(n_iter=10)
#optimizer = CoordinateSearch(n_iter=20)
# this is what runs it all
po = ParslRunner(obj_func=getattr(paropt.runner.parsl, "timeCmd"),
obj_func_params={'timeout': 15},
optimizer=optimizer,
storage=storage,
experiment=experiment_inst,
logs_root_dir='./myTestLogs')
po.run()
def setupAWS():
# launch a small parsl job on AWS to initialize parsl's AWS VPC stuff
# If run successfully, it will create the awsproviderstate.json file on host in paropt-service/config/
# Needs to be run each time the AWS credentials are changed for the server
# Intended to be used with a `docker run ...` command before running production server
import os
import paropt
from paropt.runner import ParslRunner
from paropt.storage import RelationalDB
from paropt.optimizer import BayesianOptimizer, GridSearch
from paropt.runner.parsl import timeCommand
from paropt.storage.entities import Parameter, PARAMETER_TYPE_INT, Experiment, LocalCompute, EC2Compute
container_state_file_dir = os.getenv("CONTAINER_STATE_FILE_DIR")
if not container_state_file_dir:
raise Exception(
"Missing required env var CONTAINER_STATE_FILE_DIR which is used for copying awsproviderstate.json to host"
)
paropt.setConsoleLogger()
command_template_string = """
#! /bin/bash
sleep ${myParam}
"""
experiment_inst = Experiment(
tool_name='tmptool',
parameters=[
Parameter(name="myParam",
type=PARAMETER_TYPE_INT,
minimum=0,
maximum=10),
],
command_template_string=command_template_string,
compute=EC2Compute(
type='ec2',
instance_model=
"c4.large", # using c5 b/c previously had trouble with t2 spot instances
instance_family="c4",
ami=
"ami-0257427d05c8c18ac" # parsl base ami - preinstalled apt packages
))
# use an ephemeral database
storage = RelationalDB(
'sqlite',
'',
'',
'',
'tmpSqliteDB',
)
# run simple bayes opt
bayesian_optimizer = BayesianOptimizer(
n_init=1,
n_iter=1,
)
po = ParslRunner(parsl_app=timeCommand,
optimizer=bayesian_optimizer,
storage=storage,
experiment=experiment_inst,
logs_root_dir='/var/log/paropt')
po.run(debug=True)
po.cleanup()
# print result
print(po.run_result)
# move the awsproviderstate file into expected directory
from shutil import copyfile
copyfile("awsproviderstate.json",
f'{container_state_file_dir}/awsproviderstate.json')