def getAWSConfig():
return Config(
executors=[
HighThroughputExecutor(
label='htex_local',
address=public_ip,
worker_port_range=(54000, 54050),
interchange_port_range=(54051, 54100),
cores_per_worker=1,
max_workers=1,
provider=AWSProvider(
image_id=
'ami-073e3f122e47832bf', # image with bio tools installed
worker_init=
'pip3 install git+https://[email protected]/macintoshpie/paropt',
region='us-east-2',
key_name='testKeyPair',
state_file='/etc/awsproviderstate.json',
nodes_per_block=1,
init_blocks=1,
max_blocks=1,
min_blocks=0,
walltime='01:00:00',
),
)
],
strategy=None,
)
from parsl.config import Config
from parsl.providers import AWSProvider
from parsl.executors import HighThroughputExecutor
config = Config(
executors=[
HighThroughputExecutor(
label='ec2_single_node',
provider=AWSProvider(
# Specify your EC2 AMI id
'YOUR_AMI_ID',
# Specify the AWS region to provision from
# eg. us-east-1
region='YOUR_AWS_REGION',
# Specify the name of the key to allow ssh access to nodes
key_name='YOUR_KEY_NAME',
profile="default",
state_file='awsproviderstate.json',
nodes_per_block=1,
init_blocks=1,
max_blocks=1,
min_blocks=0,
walltime='01:00:00',
),
)
],
)
# If you are a developer running tests, make sure to update parsl/tests/configs/user_opts.py
# If you are a user copying-and-pasting this as an example, make sure to either
# 1) create a local `user_opts.py`, or
# 2) delete the user_opts import below and replace all appearances of `user_opts` with the literal value
# (i.e., user_opts['swan']['username'] -> 'your_username')
from .user_opts import user_opts
config = Config(
executors=[
IPyParallelExecutor(
label='ec2_bad_spot',
workers_per_node=1,
provider=AWSProvider(
user_opts['ec2']['image_id'],
region=user_opts['ec2']['region'],
key_name=user_opts['ec2']['key_name'],
profile="default",
state_file='awsproviderstate.json',
spot_max_bid='0.001',
nodes_per_block=1,
init_blocks=1,
max_blocks=1,
min_blocks=0,
walltime='00:25:00',
),
controller=Controller(public_ip=user_opts['public_ip']),
)
],
run_dir=get_rundir(),
)
def parslConfigFromCompute(compute):
"""Given a Compute instance, return a setup parsl configuration"""
if isinstance(compute, EC2Compute):
# NOTE: Assumes the paropt is being run on an EC2 instance with access to metadata service
try:
public_ip = getAWSPublicIP()
# get the required environment variables
required_env_vars = [
"PAROPT_AWS_REGION", "PAROPT_AWS_KEY_NAME",
"PAROPT_AWS_STATE_FILE", "PAROPT_AWS_IAM_INSTANCE_PROFILE_ARN"
]
env_vars = {
varname.replace('PAROPT_AWS_', '').lower(): os.getenv(varname)
for varname in required_env_vars
}
missing_vars = [
varname for varname, value in env_vars.items() if value == None
]
if missing_vars:
raise Exception(
"Missing required environment variables for running parsl with AWS:\n{}"
.format(missing_vars))
parsl_config = Config(
executors=[
HighThroughputExecutor(
label='htex_local',
address=public_ip,
worker_port_range=(54000, 54050),
interchange_port_range=(54051, 54100),
cores_per_worker=1,
max_workers=1,
provider=AWSProvider(
image_id=compute.ami,
instance_type=compute.instance_model,
worker_init=
'pip3 install git+https://[email protected]/globus-labs/ParaOpt@Chaofeng_modification', # git+https://[email protected]/chaofengwu/paropt',#git+https://[email protected]/macintoshpie/paropt',
nodes_per_block=1,
init_blocks=1,
max_blocks=1,
min_blocks=0,
walltime='24:00:00',
spot_max_bid=2.0,
**env_vars),
)
],
strategy=None,
)
return parsl_config
except KeyError as e:
logger.error('Failed initializing aws config: {}'.format(e))
raise e
except (HTTPError, URLError, OSError) as e:
logger.error('Request to metadata service failed: {}'.format(e))
raise e
elif isinstance(compute, LocalCompute):
return Config(executors=[
ThreadPoolExecutor(max_threads=8, label='local_threads')
])
elif isinstance(compute, PBSProCompute):
# NOTE: Assumes the paropt is being run on an PBS node with access to metadata service
try:
parsl_config = Config(
executors=[
HighThroughputExecutor(
label="htex",
heartbeat_period=15,
heartbeat_threshold=120,
worker_debug=True,
max_workers=4,
address=address_by_interface('ib0'),
provider=PBSProProvider(
launcher=MpiRunLauncher(),
# PBS directives (header lines): for array jobs pass '-J' option
# scheduler_options='#PBS -J 1-10',
scheduler_options=compute.scheduler_options,
# Command to be run before starting a worker, such as:
# 'module load Anaconda; source activate parsl_env'.
worker_init=compute.worker_init,
# number of compute nodes allocated for each block
nodes_per_block=1,
min_blocks=1,
max_blocks=5,
cpus_per_node=compute.cpus_per_node,
# medium queue has a max walltime of 24 hrs
walltime=compute.walltime),
),
],
monitoring=MonitoringHub(
hub_address=address_by_interface('ib0'),
hub_port=55055,
resource_monitoring_interval=10,
),
strategy='simple',
retries=3,
app_cache=True,
checkpoint_mode='task_exit')
return parsl_config
except KeyError as e:
logger.error('Failed initializing PBSPro config: {}'.format(e))
raise e
else:
raise Exception('Unknown Compute type')
from parsl.config import Config
from parsl.executors.ipp import IPyParallelExecutor
from parsl.executors.ipp_controller import Controller
# This is an example config, make sure to
# replace the specific values below with the literal values
# (e.g., 'USERNAME' -> 'your_username')
config = Config(
executors=[
IPyParallelExecutor(
label='ec2_single_node',
provider=AWSProvider(
'image_id', # Please replace image_id with your image id, e.g., 'ami-82f4dae7'
region='us-east-1', # Please replace region with your region
key_name='KEY', # Please replace KEY with your key name
profile="default",
state_file='awsproviderstate.json',
nodes_per_block=1,
tasks_per_node=2,
init_blocks=1,
max_blocks=1,
min_blocks=0,
walltime='01:00:00',
),
controller=Controller(public_ip='PUBLIC_IP'), # Please replace PUBLIC_IP with your public ip
)
],
)
def parslConfigFromCompute(compute):
"""Given a Compute instance, return a setup parsl configuration"""
if isinstance(compute, EC2Compute):
# NOTE: Assumes the paropt is being run on an EC2 instance with access to metadata service
try:
public_ip = getAWSPublicIP()
# get the required environment variables
required_env_vars = [
"PAROPT_AWS_REGION", "PAROPT_AWS_KEY_NAME",
"PAROPT_AWS_STATE_FILE", "PAROPT_AWS_IAM_INSTANCE_PROFILE_ARN"
]
env_vars = {
varname.replace('PAROPT_AWS_', '').lower(): os.getenv(varname)
for varname in required_env_vars
}
missing_vars = [
varname for varname, value in env_vars.items() if value == None
]
if missing_vars:
raise Exception(
"Missing required environment variables for running parsl with AWS:\n{}"
.format(missing_vars))
parsl_config = Config(
executors=[
HighThroughputExecutor(
label='htex_local',
address=public_ip,
worker_port_range=(54000, 54050),
interchange_port_range=(54051, 54100),
cores_per_worker=1,
max_workers=1,
provider=AWSProvider(
image_id=compute.ami,
instance_type=compute.instance_model,
worker_init=
'pip3 install git+https://[email protected]/macintoshpie/paropt',
nodes_per_block=1,
init_blocks=1,
max_blocks=1,
min_blocks=0,
walltime='01:00:00',
spot_max_bid=2.0,
**env_vars),
)
],
strategy=None,
)
return parsl_config
except KeyError as e:
logger.error('Failed initializing aws config: {}'.format(e))
raise e
except (HTTPError, URLError, OSError) as e:
logger.error('Request to metadata service failed: {}'.format(e))
raise e
elif isinstance(compute, LocalCompute):
return Config(executors=[
ThreadPoolExecutor(max_threads=8, label='local_threads')
])
else:
raise Exception('Unknown Compute type')
from parsl.config import Config
from parsl.providers import AWSProvider
from parsl.executors import HighThroughputExecutor
from parsl.addresses import address_by_query
config = Config(executors=[
HighThroughputExecutor(
label='ec2_htex_single_node',
address=address_by_query(),
provider=AWSProvider(
image_id='YOUR_AMI_ID',
region='us-east-1',
key_name='YOUR_KEY_NAME',
profile='default',
state_file='awsproviderstate.json',
nodes_per_block=1,
init_blocks=1,
walltime='01:00:00',
),
)
], )
