def local_setup():
threads_config = Config(
executors=[
HighThroughputExecutor(
label="theta_htex",
# worker_debug=True,
cores_per_worker=4,
provider=CobaltProvider(
queue='debug-flat-quad',
account='CSC249ADCD01',
launcher=AprunLauncher(overrides="-d 64"),
worker_init='source activate parsl-issues',
init_blocks=1,
max_blocks=1,
min_blocks=1,
nodes_per_block=4,
cmd_timeout=60,
walltime='00:10:00',
),
)
],
monitoring=MonitoringHub(hub_port=55055,
logging_level=logging.DEBUG,
resource_monitoring_interval=10),
strategy=None)
parsl.load(threads_config)
def local_setup():
threads_config = Config(
executors=[ThreadPoolExecutor(label='threads', max_threads=4)],
monitoring=MonitoringHub(hub_address="127.0.0.1",
hub_port=55055,
logging_level=logging.INFO,
resource_monitoring_interval=10))
parsl.load(threads_config)
import os
config = Config(
executors=[
HighThroughputExecutor(
cores_per_worker=4,
mem_per_worker=40,
max_workers=4,
worker_debug=True,
address=address_by_hostname(),
provider=SlurmProvider(
'daenerys',
worker_init=("source activate /cephfs/users/jbreynier/conda/parsl_env2 ; "
"export PYTHONPATH='{}:{{PYTHONPATH}}'").format(os.getcwd()),
init_blocks=1,
max_blocks=10,
min_blocks=0,
nodes_per_block=1,
walltime='99:00:00',
scheduler_options='#SBATCH --exclude=kg15-11 --cpus-per-task=16 --mem=160gb --time=99:00:00',
),
),
],
monitoring=MonitoringHub(
hub_address=address_by_hostname(),
hub_port=55055,
monitoring_debug=False,
resource_monitoring_interval=10,
),
checkpoint_mode='task_exit'
)
def parslConfigFromCompute(compute):
"""Given a Compute instance, return a setup parsl configuration"""
if isinstance(compute, EC2Compute):
# NOTE: Assumes the paropt is being run on an EC2 instance with access to metadata service
try:
public_ip = getAWSPublicIP()
# get the required environment variables
required_env_vars = [
"PAROPT_AWS_REGION", "PAROPT_AWS_KEY_NAME",
"PAROPT_AWS_STATE_FILE", "PAROPT_AWS_IAM_INSTANCE_PROFILE_ARN"
]
env_vars = {
varname.replace('PAROPT_AWS_', '').lower(): os.getenv(varname)
for varname in required_env_vars
}
missing_vars = [
varname for varname, value in env_vars.items() if value == None
]
if missing_vars:
raise Exception(
"Missing required environment variables for running parsl with AWS:\n{}"
.format(missing_vars))
parsl_config = Config(
executors=[
HighThroughputExecutor(
label='htex_local',
address=public_ip,
worker_port_range=(54000, 54050),
interchange_port_range=(54051, 54100),
cores_per_worker=1,
max_workers=1,
provider=AWSProvider(
image_id=compute.ami,
instance_type=compute.instance_model,
worker_init=
'pip3 install git+https://[email protected]/globus-labs/ParaOpt@Chaofeng_modification', # git+https://[email protected]/chaofengwu/paropt',#git+https://[email protected]/macintoshpie/paropt',
nodes_per_block=1,
init_blocks=1,
max_blocks=1,
min_blocks=0,
walltime='24:00:00',
spot_max_bid=2.0,
**env_vars),
)
],
strategy=None,
)
return parsl_config
except KeyError as e:
logger.error('Failed initializing aws config: {}'.format(e))
raise e
except (HTTPError, URLError, OSError) as e:
logger.error('Request to metadata service failed: {}'.format(e))
raise e
elif isinstance(compute, LocalCompute):
return Config(executors=[
ThreadPoolExecutor(max_threads=8, label='local_threads')
])
elif isinstance(compute, PBSProCompute):
# NOTE: Assumes the paropt is being run on an PBS node with access to metadata service
try:
parsl_config = Config(
executors=[
HighThroughputExecutor(
label="htex",
heartbeat_period=15,
heartbeat_threshold=120,
worker_debug=True,
max_workers=4,
address=address_by_interface('ib0'),
provider=PBSProProvider(
launcher=MpiRunLauncher(),
# PBS directives (header lines): for array jobs pass '-J' option
# scheduler_options='#PBS -J 1-10',
scheduler_options=compute.scheduler_options,
# Command to be run before starting a worker, such as:
# 'module load Anaconda; source activate parsl_env'.
worker_init=compute.worker_init,
# number of compute nodes allocated for each block
nodes_per_block=1,
min_blocks=1,
max_blocks=5,
cpus_per_node=compute.cpus_per_node,
# medium queue has a max walltime of 24 hrs
walltime=compute.walltime),
),
],
monitoring=MonitoringHub(
hub_address=address_by_interface('ib0'),
hub_port=55055,
resource_monitoring_interval=10,
),
strategy='simple',
retries=3,
app_cache=True,
checkpoint_mode='task_exit')
return parsl_config
except KeyError as e:
logger.error('Failed initializing PBSPro config: {}'.format(e))
raise e
else:
raise Exception('Unknown Compute type')
worker_init=os.environ['PT_ENVSETUP'], # Initial ENV setup
))
## This is based on the *default* executor (*DO NOT USE* due to this
## executor is not recommended by Yadu)
coriLogin = ThreadPoolExecutor(label='coriLogin',
managed=True,
max_threads=2,
storage_access=[],
thread_name_prefix='',
working_dir=None)
###################################################
###################################################
###################################################
##
## Finally, assemble the full Parsl configuration
## [Be sure to specify your needed executor(s)]
config = Config(app_cache=True,
checkpoint_mode='task_exit',
executors=[knlMj],
monitoring=MonitoringHub(
hub_address=address_by_hostname(),
hub_port=55055,
logging_level=logging.INFO,
resource_monitoring_interval=60,
),
retries=2)
# TODO: proper boolean switch here to switch between checkpointing and
# monitoring, as they do not work together at the moment.
# - see https://github.com/Parsl/parsl/issues/1014
config = parsl.config.Config(
executors=[
parsl.executors.ThreadPoolExecutor(label="management", max_threads=20),
parsl.executors.ThreadPoolExecutor(label="heavy", max_threads=3),
],
# monitoring config from
# https://parsl.readthedocs.io/en/latest/userguide/monitoring.html
# modified to add hub_port - see https://github.com/Parsl/parsl/issues/1010
monitoring=MonitoringHub(hub_address=address_by_hostname(),
logging_level=logging.INFO,
resource_monitoring_interval=10,
hub_port=30733))
# config.checkpoint_mode = 'task_exit'
REPO_BASE = "REPO"
logger.info("Getting checkpoint files")
config.checkpoint_files = parsl.utils.get_all_checkpoints()
logger.info("Checkpoint files: {}".format(config.checkpoint_files))
class RepoInfo:
def __init__(self, repo_base, rerun=None):
self.repo_base = repo_base
self.rerun = rerun
from parsl import *
# from parsl.monitoring.db_logger import MonitoringConfig
from parsl.monitoring.monitoring import MonitoringHub
from parsl.config import Config
from parsl.executors import ThreadPoolExecutor
import logging
# parsl.set_stream_logger()
threads_config = Config(
executors=[ThreadPoolExecutor(label='threads', max_threads=4)],
monitoring=MonitoringHub(
hub_address="127.0.0.1",
hub_port=55055,
logging_level=logging.INFO,
resource_monitoring_interval=10,
))
dfk = DataFlowKernel(config=threads_config)
@App('python', dfk)
def sleeper(dur=25):
import time
time.sleep(dur)
@App('python', dfk)
def cpu_stress(dur=30):
import time
s = 0
heartbeat_period=15,
heartbeat_threshold=120,
worker_debug=True,
max_workers=4,
address=address_by_interface('ib0'),
provider=PBSProProvider(
launcher=MpiRunLauncher(),
# PBS directives (header lines): for array jobs pass '-J' option
scheduler_options='#PBS -J 1-10',
# Command to be run before starting a worker, such as:
# 'module load Anaconda; source activate parsl_env'.
worker_init='',
# number of compute nodes allocated for each block
nodes_per_block=3,
min_blocks=3,
max_blocks=5,
cpus_per_node=24,
# medium queue has a max walltime of 24 hrs
walltime='24:00:00'),
),
],
monitoring=MonitoringHub(
hub_address=address_by_interface('ib0'),
hub_port=55055,
resource_monitoring_interval=10,
),
strategy='simple',
retries=3,
app_cache=True,
checkpoint_mode='task_exit')
# medium queue
nodes = 4
walltime = '24:00:00'
max_blocks = 3
config = Config(executors=[
HighThroughputExecutor(
label="htex",
cores_per_worker=3,
address=address_by_interface('ib0'),
provider=PBSProProvider(
launcher=MpiRunLauncher(),
scheduler_options='#PBS -P 11001079',
worker_init='module load gromacs; export PYTHONPATH={}:$PYTHONPATH'
.format(os.path.abspath('.')),
nodes_per_block=nodes,
max_blocks=max_blocks,
cpus_per_node=24,
walltime=walltime),
),
],
monitoring=MonitoringHub(
hub_address=address_by_interface('ib0'),
hub_port=55055,
logging_level=logging.INFO,
resource_monitoring_interval=10,
),
strategy='simple',
retries=3)
def workflow_config(name,
nodes,
cores_per_node=24,
interval=30,
monitor=False):
import parsl
from parsl.config import Config
from parsl.channels import LocalChannel
from parsl.launchers import SrunLauncher
from parsl.providers import LocalProvider
from parsl.addresses import address_by_interface
from parsl.executors import HighThroughputExecutor
from parsl.monitoring.monitoring import MonitoringHub
parsl.set_stream_logger()
parsl.set_file_logger('script.output', level=logging.DEBUG)
logging.info('Configuring Parsl Workflow Infrastructure')
#Read where datasets are...
env_str = str()
with open('parsl.env', 'r') as reader:
env_str = reader.read()
logging.info(f'Task Environment {env_str}')
mon_hub = MonitoringHub(
workflow_name=name,
hub_address=address_by_interface('ib0'),
hub_port=60001,
resource_monitoring_enabled=True,
monitoring_debug=False,
resource_monitoring_interval=interval,
) if monitor else None
config = Config(
executors=[
HighThroughputExecutor(
label=name,
# Optional: The network interface on node 0 which compute nodes can communicate with.
# address=address_by_interface('enp4s0f0' or 'ib0')
address=address_by_interface('ib0'),
# one worker per manager / node
max_workers=cores_per_node,
provider=LocalProvider(
channel=LocalChannel(script_dir='.'),
# make sure the nodes_per_block matches the nodes requested in the submit script in the next step
nodes_per_block=nodes,
# make sure
launcher=SrunLauncher(overrides=f'-c {cores_per_node}'),
cmd_timeout=120,
init_blocks=1,
max_blocks=1,
worker_init=env_str,
),
)
],
monitoring=mon_hub,
strategy=None,
)
logging.info('Loading Parsl Config')
parsl.load(config)
return
