def parslConfigFromCompute(compute):
"""Given a Compute instance, return a setup parsl configuration"""
if isinstance(compute, EC2Compute):
# NOTE: Assumes the paropt is being run on an EC2 instance with access to metadata service
try:
public_ip = getAWSPublicIP()
# get the required environment variables
required_env_vars = [
"PAROPT_AWS_REGION", "PAROPT_AWS_KEY_NAME",
"PAROPT_AWS_STATE_FILE", "PAROPT_AWS_IAM_INSTANCE_PROFILE_ARN"
]
env_vars = {
varname.replace('PAROPT_AWS_', '').lower(): os.getenv(varname)
for varname in required_env_vars
}
missing_vars = [
varname for varname, value in env_vars.items() if value == None
]
if missing_vars:
raise Exception(
"Missing required environment variables for running parsl with AWS:\n{}"
.format(missing_vars))
parsl_config = Config(
executors=[
HighThroughputExecutor(
label='htex_local',
address=public_ip,
worker_port_range=(54000, 54050),
interchange_port_range=(54051, 54100),
cores_per_worker=1,
max_workers=1,
provider=AWSProvider(
image_id=compute.ami,
instance_type=compute.instance_model,
worker_init=
'pip3 install git+https://[email protected]/globus-labs/ParaOpt@Chaofeng_modification', # git+https://[email protected]/chaofengwu/paropt',#git+https://[email protected]/macintoshpie/paropt',
nodes_per_block=1,
init_blocks=1,
max_blocks=1,
min_blocks=0,
walltime='24:00:00',
spot_max_bid=2.0,
**env_vars),
)
],
strategy=None,
)
return parsl_config
except KeyError as e:
logger.error('Failed initializing aws config: {}'.format(e))
raise e
except (HTTPError, URLError, OSError) as e:
logger.error('Request to metadata service failed: {}'.format(e))
raise e
elif isinstance(compute, LocalCompute):
return Config(executors=[
ThreadPoolExecutor(max_threads=8, label='local_threads')
])
else:
raise Exception('Unknown Compute type')
nproc = 36
sched_opts = '''
#SBATCH --cpus-per-task=%d
''' % (nproc)
slurm_htex = Config(
executors=[
HighThroughputExecutor(
label="coffea_parsl_slurm",
address=address_by_hostname(),
prefetch_capacity=0,
max_workers=nproc,
provider=SlurmProvider(
channel=LocalChannel(script_dir='parsl_slurm'),
launcher=SrunLauncher(),
max_blocks=(args.ncpu)+5,
init_blocks=args.ncpu,
partition='all',
scheduler_options=sched_opts, # Enter scheduler_options if needed
worker_init=wrk_init, # Enter worker_init if needed
walltime='00:120:00'
),
)
],
retries=20,
)
dfk = parsl.load(slurm_htex)
else:
config = Config(executors=[ThreadPoolExecutor(max_threads=args.ncpu)])
parsl.load(config)
from parsl import App, DataFlowKernel
# from parsl.monitoring.db_logger import MonitoringConfig
from parsl.monitoring.monitoring import MonitoringHub
from parsl.config import Config
from parsl.executors import ThreadPoolExecutor
import logging
# parsl.set_stream_logger()
threads_config = Config(
executors=[ThreadPoolExecutor(
label='threads',
max_threads=4)
],
monitoring=MonitoringHub(
hub_address="127.0.0.1",
hub_port=55055,
logging_level=logging.INFO,
resource_monitoring_interval=10,
)
)
dfk = DataFlowKernel(config=threads_config)
@App('python', dfk)
def sleeper(dur=25):
import time
time.sleep(dur)
