def parslConfigFromCompute(compute):
"""Given a Compute instance, return a setup parsl configuration"""
if isinstance(compute, EC2Compute):
# NOTE: Assumes the paropt is being run on an EC2 instance with access to metadata service
try:
public_ip = getAWSPublicIP()
# get the required environment variables
required_env_vars = [
"PAROPT_AWS_REGION", "PAROPT_AWS_KEY_NAME",
"PAROPT_AWS_STATE_FILE", "PAROPT_AWS_IAM_INSTANCE_PROFILE_ARN"
]
env_vars = {
varname.replace('PAROPT_AWS_', '').lower(): os.getenv(varname)
for varname in required_env_vars
}
missing_vars = [
varname for varname, value in env_vars.items() if value == None
]
if missing_vars:
raise Exception(
"Missing required environment variables for running parsl with AWS:\n{}"
.format(missing_vars))
parsl_config = Config(
executors=[
HighThroughputExecutor(
label='htex_local',
address=public_ip,
worker_port_range=(54000, 54050),
interchange_port_range=(54051, 54100),
cores_per_worker=1,
max_workers=1,
provider=AWSProvider(
image_id=compute.ami,
instance_type=compute.instance_model,
worker_init=
'pip3 install git+https://[email protected]/globus-labs/ParaOpt@Chaofeng_modification', # git+https://[email protected]/chaofengwu/paropt',#git+https://[email protected]/macintoshpie/paropt',
nodes_per_block=1,
init_blocks=1,
max_blocks=1,
min_blocks=0,
walltime='24:00:00',
spot_max_bid=2.0,
**env_vars),
)
],
strategy=None,
)
return parsl_config
except KeyError as e:
logger.error('Failed initializing aws config: {}'.format(e))
raise e
except (HTTPError, URLError, OSError) as e:
logger.error('Request to metadata service failed: {}'.format(e))
raise e
elif isinstance(compute, LocalCompute):
return Config(executors=[
ThreadPoolExecutor(max_threads=8, label='local_threads')
])
else:
raise Exception('Unknown Compute type')
HighThroughputExecutor(
label='comet_htex',
worker_debug=True,
address='js-17-185.jetstream-cloud.org',
max_workers=1,
cores_per_worker=24,
worker_logdir_root='/home/aymen/parsl_scripts',
#address=address_by_query(),
interchange_address='comet-ln2.sdsc.edu',
interchange_port_range=(50100, 50400),
#client_address = "129.114.17.185",
worker_port_range=(50500, 51000),
provider=SlurmProvider(
'debug',
channel=SSHChannel(
hostname='comet-ln2.sdsc.edu',
username=
'******', # Please replace USERNAME with your username
password='******',
script_dir=
'/home/aymen/parsl_scripts', # Please replace USERNAME with your username
),
# launcher=SrunLauncher(),
scheduler_options='', # Input your scheduler_options if needed
#worker_init='conda activate /oasis/projects/nsf/unc100/aymen/anaconda3/envs/parsl-env', # Input your worker_init if needed
worker_init='source /home/aymen/ve/parsl-env/bin/activate',
walltime="00:10:00",
init_blocks=1,
max_blocks=1,
nodes_per_block=1,
parallelism=24,
),
working_dir="/home/aymen/parsl_scripts",
#client_address = "129.114.17.185",
#worker_port_range=(54000, 55000),
#interchange_address = "js-17-185.jetstream-cloud.org"
#storage_access=[GlobusScheme(
# endpoint_uuid='de463f97-6d04-11e5-ba46-22000b92c6ec',
# endpoint_path='/',
# local_path='/')],
)
from parsl.providers import CobaltProvider
from parsl.launchers import AprunLauncher
from parsl.executors import HighThroughputExecutor
config = Config(
executors=[
HighThroughputExecutor(
label='theta_local_htex_multinode',
max_workers=4,
provider=CobaltProvider(
queue='YOUR_QUEUE',
account='YOUR_ACCOUNT',
launcher=AprunLauncher(overrides="-d 64"),
walltime='00:30:00',
nodes_per_block=2,
init_blocks=1,
min_blocks=1,
max_blocks=1,
# string to prepend to #COBALT blocks in the submit
# script to the scheduler eg: '#COBALT -t 50'
scheduler_options='',
# Command to be run before starting a worker, such as:
# 'module load Anaconda; source activate parsl_env'.
worker_init='',
cmd_timeout=120,
),
)
],
)
from datetime import datetime
import csv
import parsl
from parsl.app.app import python_app, bash_app
from parsl.config import Config
from parsl.providers import GridEngineProvider
from parsl.executors import HighThroughputExecutor
from parsl.executors import IPyParallelExecutor
from parsl.executors import ThreadPoolExecutor
from parsl.addresses import address_by_route, address_by_query, address_by_hostname
config = Config(executors=[
HighThroughputExecutor(worker_debug=True,
address=address_by_route(),
provider=GridEngineProvider(walltime='100:00:00',
init_blocks=1,
max_blocks=20),
label="workers"),
ThreadPoolExecutor(label="login", max_threads=20)
], )
parsl.set_stream_logger()
parsl.load(config)
from data_generation import generate_data
proteomefile = sys.argv[1]
directory = f'/home/users/ellenrichards/{sys.argv[2]}/'
threshold = 1000
if not os.path.isdir(directory):
executors=[
HighThroughputExecutor(
label="theta_funcx",
#worker_debug=True,
max_workers=tasks_per_node, # Experimental
#suppress_failure=True, # Experimental
heartbeat_period=60,
heartbeat_threshold=600,
cores_per_worker=args.cores_per_worker,
address=address_by_hostname(),
#container_image=os.path.expanduser("~/sing-run.simg"),
container_image='/tmp/sing-run.simg',
worker_mode="singularity_reuse",
#worker_mode="no_container",
provider=CobaltProvider(
queue=args.queue,
#account='ExM',
account='CSC249ADCD01',
launcher=AprunLauncher(overrides="-d 64"),
scheduler_options='',
# worker_init='source ~/move_image.sh\nsource activate funcx-test'.format(os.getenv('PWD')),
worker_init=
'source activate funcx-test\naprun -n {} -N 1 /bin/bash ~/move_image.sh'
.format(nodes_per_block),
init_blocks=1,
max_blocks=1,
min_blocks=1,
nodes_per_block=nodes_per_block,
walltime=walltime,
cmd_timeout=60),
)
],
docker_image = "opensciencegrid/osgvo-el6"
transfer_output_files = coffea_parsl_condor
''' #% (nproc, ) # twoGB*nproc,
#RequestMemory = %d
#RequestCpus = %d
#RequestDisk = 1048576
xfer_files = ['%s/.local' % (os.environ['HOME'], ), '/tmp/%s' % (x509_proxy, )]
config = Config(
executors=[
HighThroughputExecutor(
label="coffea_parsl_condor",
address=address_by_hostname(),
prefetch_capacity=0,
cores_per_worker=1,
max_workers=nproc,
worker_logdir_root='./',
provider=CondorProvider(init_blocks=8,
max_blocks=200,
nodes_per_block=1,
worker_init=wrk_init,
transfer_input_files=xfer_files,
scheduler_options=condor_cfg),
)
],
strategy=None,
)
from parsl.config import Config
from parsl.providers import CondorProvider
from parsl.executors import HighThroughputExecutor
from parsl.addresses import address_by_query
config = Config(executors=[
HighThroughputExecutor(
label='OSG_HTEX',
address=address_by_query(),
max_workers=1,
provider=CondorProvider(
nodes_per_block=1,
init_blocks=4,
max_blocks=4,
# This scheduler option string ensures that the compute nodes provisioned
# will have modules
scheduler_options=
'Requirements = OSGVO_OS_STRING == "RHEL 6" && Arch == "X86_64" && HAS_MODULES == True',
# Command to be run before starting a worker, such as:
# 'module load Anaconda; source activate parsl_env'.
worker_init='',
walltime="00:20:00",
),
)
])
from typing import Any, Dict
user_opts = {'adhoc':
{'username': '******',
'script_dir': 'YOUR_SCRIPT_DIR',
'remote_hostnames': ['REMOTE_HOST_URL_1', 'REMOTE_HOST_URL_2']
}
} # type: Dict[str, Dict[str, Any]]
config = Config(
executors=[
HighThroughputExecutor(
label='remote_htex',
max_workers=2,
address=address_by_query(),
worker_logdir_root=user_opts['adhoc']['script_dir'],
provider=AdHocProvider(
# Command to be run before starting a worker, such as:
# 'module load Anaconda; source activate parsl_env'.
worker_init='',
channels=[SSHChannel(hostname=m,
username=user_opts['adhoc']['username'],
script_dir=user_opts['adhoc']['script_dir'],
) for m in user_opts['adhoc']['remote_hostnames']]
)
)
],
# AdHoc Clusters should not be setup with scaling strategy.
strategy=None,
)
from parsl.config import Config
from parsl.executors import HighThroughputExecutor
n_managers = 2
config = Config(
executors=[
HighThroughputExecutor(
address=address_by_interface('bond0.144'),
poll_period=1,
heartbeat_period=1,
heartbeat_threshold=2,
label="htex_local",
worker_mode="no_container",
worker_ports=(53531, 53532),
# worker_debug=True,
cores_per_worker=1,
max_workers=4,
provider=LocalProvider(
channel=LocalChannel(),
init_blocks=0,
max_blocks=1,
# tasks_per_node=1, # For HighThroughputExecutor, this option should in most cases be 1
launcher=SingleNodeLauncher(),
),
)
],
retries=2,
strategy=None,
)
import getpass
vm_reference = {
# All fields below are required
"admin_username": '******',
"password": '******',
"vm_size": 'YOUR_VM_SIZE',
"disk_size_gb": 'YOUR_VM_DISK_SIZE',
"publisher": 'YOUR_IMAGE_PUBLISHER',
"offer": 'YOUR_VM_OS_OFFER',
"sku": 'YOUR_VM_OS_SKU',
"version": 'YOUR_VM_OS_VERSION',
}
config = Config(executors=[
HighThroughputExecutor(
label='azure_single_node',
address=address_by_query(),
provider=AzureProvider(
vm_reference=vm_reference,
key_file='azure_key_file.json',
),
storage_access=[
HTTPInTaskStaging(),
FTPInTaskStaging(),
RSyncStaging(getpass.getuser() + "@" + address_by_query())
],
)
])
from parsl.providers import LocalProvider
from parsl.channels import LocalChannel
from parsl.config import Config
from parsl.executors import HighThroughputExecutor
config = Config(executors=[
HighThroughputExecutor(
label="htex_local",
cores_per_worker=1,
provider=LocalProvider(
channel=LocalChannel(),
init_blocks=1,
max_blocks=1,
),
)
], )
channel = SSHChannel(
hostname=hostnames[step],
username=args.unix_username,
gssapi_auth=args.gssapi,
)
if args.scheduler_name == 'slurm':
executors.append(
HighThroughputExecutor(
label=step,
worker_debug=True,
address=address_by_hostname(),
cores_per_worker=vCPUs_per_core * int(cores_per_task[step]),
provider=SlurmProvider(
args.scheduler_partition,
channel=channel,
launcher=SrunLauncher(),
nodes_per_block=node_count,
worker_init=worker_init,
init_blocks=1,
max_blocks=1,
walltime=walltimes[step],
scheduler_options=options,
move_files=False,
),
))
elif args.scheduler_name == 'grid_engine':
executors.append(
HighThroughputExecutor(
label=step,
worker_debug=True,
address=address_by_hostname(),
provider=GridEngineProvider(
from parsl.config import Config
from parsl.providers import AWSProvider
from parsl.executors import HighThroughputExecutor
from parsl.addresses import address_by_query
config = Config(executors=[
HighThroughputExecutor(
label='ec2_htex_single_node',
address=address_by_query(),
provider=AWSProvider(
image_id='YOUR_AMI_ID',
region='us-east-1',
key_name='YOUR_KEY_NAME',
profile='default',
state_file='awsproviderstate.json',
nodes_per_block=1,
init_blocks=1,
walltime='01:00:00',
),
)
], )
def theta_nwchem_config(log_dir: str,
nodes_per_nwchem: int = 2,
total_nodes: int = int(
os.environ.get("COBALT_JOBSIZE", 1)),
ml_prefetch: int = 0) -> Config:
"""Theta configuration where QC workers sit on the launch node (to be able to aprun)
and ML workers are placed on compute nodes
Args:
nodes_per_nwchem: Number of nodes per NWChem computation
log_dir: Path to store monitoring DB and parsl logs
total_nodes: Total number of nodes available. Default: COBALT_JOBSIZE
ml_prefetch: Number of tasks for ML workers to prefetch for inference
Returns:
(Config) Parsl configuration
"""
assert total_nodes % nodes_per_nwchem == 0, "NWChem node count not a multiple of nodes per task"
nwc_workers = total_nodes // nodes_per_nwchem
return Config(
executors=[
ThreadPoolExecutor(label='qc', max_threads=nwc_workers),
HighThroughputExecutor(
address=address_by_hostname(),
label="ml-inference",
max_workers=1,
prefetch_capacity=ml_prefetch,
provider=LocalProvider(
nodes_per_block=nodes_per_nwchem,
init_blocks=0,
max_blocks=total_nodes //
nodes_per_nwchem, # Limits the number of manager processes,
launcher=AprunLauncher(
overrides='-d 256 --cc depth -j 4'
), # Places worker on the compute node
worker_init='''
module load miniconda-3
conda activate /lus/theta-fs0/projects/CSC249ADCD08/edw/env
''',
),
),
HighThroughputExecutor(
address=address_by_hostname(),
label="ml-train",
max_workers=1,
prefetch_capacity=0,
provider=LocalProvider(
nodes_per_block=nodes_per_nwchem,
init_blocks=0,
max_blocks=
nwc_workers, # Limits the number of manager processes,
launcher=AprunLauncher(
overrides='-d 256 --cc depth -j 4'
), # Places worker on the compute node
worker_init='''
module load miniconda-3
conda activate /lus/theta-fs0/projects/CSC249ADCD08/edw/env
''',
),
)
],
monitoring=MonitoringHub(
hub_address=address_by_hostname(),
monitoring_debug=False,
resource_monitoring_interval=10,
logdir=log_dir,
logging_endpoint=
f'sqlite:///{os.path.join(log_dir, "monitoring.db")}'),
run_dir=log_dir,
strategy='simple',
max_idletime=15.)
def cli_run():
parser = argparse.ArgumentParser()
parser.add_argument(
"--redishost",
default="127.0.0.1",
help="Address at which the redis server can be reached")
parser.add_argument("--redisport",
default="6379",
help="Port on which redis is available")
parser.add_argument("-d",
"--debug",
action='store_true',
help="Count of apps to launch")
parser.add_argument("-m",
"--mac",
action='store_true',
help="Configure for Mac")
args = parser.parse_args()
if args.debug:
parsl.set_stream_logger()
if args.mac:
config = Config(
executors=[
ThreadPoolExecutor(label="htex"),
ThreadPoolExecutor(label="local_threads")
],
strategy=None,
)
else:
config = Config(
executors=[
HighThroughputExecutor(
label="htex",
# Max workers limits the concurrency exposed via mom node
max_workers=2,
provider=LocalProvider(
init_blocks=1,
max_blocks=1,
),
),
ThreadPoolExecutor(label="local_threads")
],
strategy=None,
)
parsl.load(config)
print(
'''This program creates an "MPI Method Server" that listens on an inputs queue and write on an output queue:
input_queue --> mpi_method_server --> queues
To send it a request, add an entry to the inputs queue:
run "pipeline-pump -p N" where N is an integer request
To access a value, remove it from the outout queue:
run "pipeline-pull" (blocking) or "pipeline-pull -t T" (T an integer) to time out after T seconds
TODO: Timeout does not work yet!
''')
# Get the queues for the method server
method_queues = MethodServerQueues(args.redishost, port=args.redisport)
# Start the method server
mms = ParslMethodServer([target_fun],
method_queues,
default_executors=['htex'])
mms.run()
from parsl.config import Config
from parsl.providers import SlurmProvider
from parsl.executors import HighThroughputExecutor
from parsl.addresses import address_by_hostname
from parsl.data_provider.globus import GlobusScheme
config = Config(
executors=[
HighThroughputExecutor(
label='Stampede2_HTEX',
address=address_by_hostname(),
provider=SlurmProvider(
nodes_per_block=2,
init_blocks=1,
min_blocks=1,
partition='YOUR_PARTITION',
# string to prepend to #SBATCH blocks in the submit
# script to the scheduler eg: '#SBATCH --constraint=knl,quad,cache'
scheduler_options='',
# Command to be run before starting a worker, such as:
# 'module load Anaconda; source activate parsl_env'.
worker_init='',
walltime='00:30:00'),
storage_access=[
GlobusScheme(
endpoint_uuid='ceea5ca0-89a9-11e7-a97f-22000a92523b',
endpoint_path='/',
local_path='/')
])
], )
def workflow_config(name,
nodes,
cores_per_node=24,
interval=30,
monitor=False):
import parsl
from parsl.config import Config
from parsl.channels import LocalChannel
from parsl.launchers import SrunLauncher
from parsl.providers import LocalProvider
from parsl.addresses import address_by_interface
from parsl.executors import HighThroughputExecutor
from parsl.monitoring.monitoring import MonitoringHub
parsl.set_stream_logger()
parsl.set_file_logger('script.output', level=logging.DEBUG)
logging.info('Configuring Parsl Workflow Infrastructure')
#Read where datasets are...
env_str = str()
with open('parsl.env', 'r') as reader:
env_str = reader.read()
logging.info(f'Task Environment {env_str}')
mon_hub = MonitoringHub(
workflow_name=name,
hub_address=address_by_interface('ib0'),
hub_port=60001,
resource_monitoring_enabled=True,
monitoring_debug=False,
resource_monitoring_interval=interval,
) if monitor else None
config = Config(
executors=[
HighThroughputExecutor(
label=name,
# Optional: The network interface on node 0 which compute nodes can communicate with.
# address=address_by_interface('enp4s0f0' or 'ib0')
address=address_by_interface('ib0'),
# one worker per manager / node
max_workers=cores_per_node,
provider=LocalProvider(
channel=LocalChannel(script_dir='.'),
# make sure the nodes_per_block matches the nodes requested in the submit script in the next step
nodes_per_block=nodes,
# make sure
launcher=SrunLauncher(overrides=f'-c {cores_per_node}'),
cmd_timeout=120,
init_blocks=1,
max_blocks=1,
worker_init=env_str,
),
)
],
monitoring=mon_hub,
strategy=None,
)
logging.info('Loading Parsl Config')
parsl.load(config)
return
WORKERS_PER_NODE = 64
parsl_config = Config(
executors=[
HighThroughputExecutor(
label='theta-htex',
max_workers=WORKERS_PER_NODE * MY_COMPUTE_NODES *
MY_COMPUTE_BLOCKS,
worker_debug=True,
address=address_by_hostname(),
provider=CobaltProvider(
queue=MY_QUEUE,
account=MY_ALLOCATION,
launcher=AprunLauncher(overrides="-d 64"),
walltime=MY_TIME,
nodes_per_block=MY_COMPUTE_NODES,
init_blocks=1,
min_blocks=1,
max_blocks=MY_COMPUTE_BLOCKS,
# string to prepend to #COBALT blocks in the submit
# script to the scheduler eg: '#COBALT -t 50'
scheduler_options='',
# Command to be run before starting a worker, such as:
worker_init='module load miniconda-3; export PATH=$PATH:{}'.
format(MY_USER_PATH),
cmd_timeout=120,
),
),
ThreadPoolExecutor(label='login-node', max_threads=8),
], )
parsl.load(parsl_config)
from parsl.config import Config
from parsl.providers import GridEngineProvider
from parsl.executors import HighThroughputExecutor
from parsl.addresses import address_by_query
config = Config(
executors=[
HighThroughputExecutor(
label='CC.IN2P3_HTEX',
address=address_by_query(),
provider=GridEngineProvider(
nodes_per_block=1,
init_blocks=1,
max_blocks=1,
# string to prepend to #SBATCH blocks in the submit
# script to the scheduler eg: '#$ -M [email protected]
scheduler_options='',
# Command to be run before starting a worker, such as:
# 'module load Anaconda; source activate parsl_env'.
worker_init='',
walltime='00:20:00',
),
)
], )
from parsl.executors.high_throughput.interchange import ManagerLost
if isinstance(exception, ManagerLost):
return 0.1
else:
return 1
slurm_htex = Config(
executors=[
HighThroughputExecutor(
label="jobs",
address=address_by_hostname(),
prefetch_capacity=0,
worker_debug=True,
provider=SlurmProvider(
channel=LocalChannel(script_dir='logs_parsl'),
launcher=SrunLauncher(),
max_blocks=args.workers,
init_blocks=args.workers,
partition='all',
# scheduler_options=sched_opts, # Enter scheduler_options if needed
worker_init=wrk_init,
walltime='03:00:00'),
),
HighThroughputExecutor(
label="merges",
address=address_by_hostname(),
prefetch_capacity=0,
worker_debug=True,
provider=SlurmProvider(
channel=LocalChannel(script_dir='logs_parsl'),
launcher=SrunLauncher(),
"""Tests related to Parsl workers being able to access their worker ID"""
from parsl.providers import LocalProvider
from parsl.channels import LocalChannel
from parsl.config import Config
from parsl.executors import HighThroughputExecutor
from parsl import python_app
import pytest
local_config = Config(
executors=[
HighThroughputExecutor(
label="htex_Local",
worker_debug=True,
max_workers=4,
provider=LocalProvider(
channel=LocalChannel(),
init_blocks=1,
max_blocks=1,
),
)
],
strategy=None,
)
@python_app
def get_worker_info():
import os
rank = int(os.environ['PARSL_WORKER_RANK'])
size = int(os.environ['PARSL_WORKER_COUNT'])
pool_id = os.environ['PARSL_WORKER_POOL_ID']
from parsl.config import Config
from parsl.executors import HighThroughputExecutor
from parsl.launchers import JsrunLauncher
from parsl.providers import LSFProvider
from parsl.addresses import address_by_interface
config = Config(
executors=[
HighThroughputExecutor(
label='Summit_HTEX',
working_dir='/gpfs/alpine/scratch/yadunan/gen011/',
address=address_by_interface(
'ib0'), # This assumes Parsl is running on login node
worker_port_range=(50000, 55000),
provider=LSFProvider(launcher=JsrunLauncher(),
walltime="00:10:00",
nodes_per_block=2,
init_blocks=1,
max_blocks=1,
worker_init="source ~/setup.sh",
project='GEN011WORKFLOW',
cmd_timeout=60),
)
], )
def main(args=None):
# Grab CLI args if not present
if args is None:
args = parse_args()
exit_callbacks = []
try:
if args["debug"]["schema"]:
print(ManagerSettings.schema_json(indent=2))
return # We're done, exit normally
except KeyError:
pass # Don't worry if schema isn't in the list
finally:
debug_args = args.pop("debug", {}) # Ensure the debug key is not present
# Construct object
settings = ManagerSettings(**args)
# Handle Skeleton Generation
if debug_args.get("skeleton", None):
class IndentListDumper(yaml.Dumper):
"""
Internal yaml Dumper to make lists indent in the output YAML
Buried inside this since its only used in "skeleton," once, and then exits. Does not need to be imported
anywhere else or accessed somehow
Based on response:
https://stackoverflow.com/questions/25108581/python-yaml-dump-bad-indentation/39681672#39681672
"""
def increase_indent(self, flow=False, indentless=False):
return super(IndentListDumper, self).increase_indent(flow, False)
skel_path = os.path.expanduser(debug_args["skeleton"])
with open(skel_path, "w") as skel:
# cast to
data = yaml.dump(json.loads(settings.json()), Dumper=IndentListDumper, default_flow_style=False)
skel.write(data)
print(
f"Skeleton Queue Manager YAML file written to {skel_path}\n"
f"Run: `qcfractal-manager --config-file={skel_path}` to start a manager with this configuration."
)
return
logger_map = {AdapterEnum.pool: "", AdapterEnum.dask: "dask_jobqueue.core", AdapterEnum.parsl: "parsl"}
if settings.common.verbose:
adapter_logger = logging.getLogger(logger_map[settings.common.adapter])
adapter_logger.setLevel("DEBUG")
logger.setLevel("DEBUG")
if settings.manager.log_file_prefix is not None:
tornado.options.options["log_file_prefix"] = settings.manager.log_file_prefix
# Clones the log to the output
tornado.options.options["log_to_stderr"] = True
tornado.log.enable_pretty_logging()
if settings.manager.test:
# Test this manager, no client needed
client = None
else:
# Connect to a specified fractal server
client = qcfractal.interface.FractalClient(
address=settings.server.fractal_uri, **settings.server.dict(skip_defaults=True, exclude={"fractal_uri"})
)
# Figure out per-task data
node_parallel_tasks = settings.common.nodes_per_task > 1 # Whether tasks are node-parallel
if node_parallel_tasks:
supported_adapters = ["parsl"]
if settings.common.adapter not in supported_adapters:
raise ValueError("Node-parallel jobs are only supported with {} adapters".format(supported_adapters))
# Node-parallel tasks use all cores on a worker
cores_per_task = settings.common.cores_per_worker
memory_per_task = settings.common.memory_per_worker
if settings.common.tasks_per_worker > 1:
raise ValueError(">1 task per node and >1 node per tasks are mutually-exclusive")
else:
cores_per_task = settings.common.cores_per_worker // settings.common.tasks_per_worker
memory_per_task = settings.common.memory_per_worker / settings.common.tasks_per_worker
if cores_per_task < 1:
raise ValueError("Cores per task must be larger than one!")
if settings.common.adapter == "pool":
from multiprocessing import Pool, set_start_method
set_start_method("spawn")
# Error if the number of nodes per jobs is more than 1
if settings.common.nodes_per_job > 1:
raise ValueError("Pool adapters only run on a single local node")
queue_client = Pool(processes=settings.common.tasks_per_worker, initializer=_initialize_signals_process_pool)
elif settings.common.adapter == "dask":
dask_settings = settings.dask.dict(skip_defaults=True)
# Checks
if "extra" not in dask_settings:
dask_settings["extra"] = []
if QCA_RESOURCE_STRING not in dask_settings["extra"]:
dask_settings["extra"].append(QCA_RESOURCE_STRING)
# Scheduler opts
scheduler_opts = settings.cluster.scheduler_options.copy()
# Error if the number of nodes per jobs is more than 1
if settings.common.nodes_per_job > 1:
raise NotImplementedError("Support for >1 node per job is not yet supported by QCFractal + Dask")
# TODO (wardlt): Implement multinode jobs in Dask
_cluster_loaders = {
"slurm": "SLURMCluster",
"pbs": "PBSCluster",
"moab": "MoabCluster",
"sge": "SGECluster",
"lsf": "LSFCluster",
}
dask_exclusivity_map = {
"slurm": "--exclusive",
"pbs": "-n",
"moab": "-n", # Less sure about this one
"sge": "-l exclusive=true",
"lsf": "-x",
}
if settings.cluster.node_exclusivity and dask_exclusivity_map[settings.cluster.scheduler] not in scheduler_opts:
scheduler_opts.append(dask_exclusivity_map[settings.cluster.scheduler])
# Create one construct to quickly merge dicts with a final check
dask_construct = {
"name": "QCFractal_Dask_Compute_Executor",
"cores": settings.common.cores_per_worker,
"memory": str(settings.common.memory_per_worker) + "GB",
"processes": settings.common.tasks_per_worker, # Number of workers to generate == tasks in this construct
"walltime": settings.cluster.walltime,
"job_extra": scheduler_opts,
"env_extra": settings.cluster.task_startup_commands,
**dask_settings,
}
try:
# Import the dask things we need
import dask_jobqueue
from dask.distributed import Client
cluster_module = cli_utils.import_module(
"dask_jobqueue", package=_cluster_loaders[settings.cluster.scheduler]
)
cluster_class = getattr(cluster_module, _cluster_loaders[settings.cluster.scheduler])
if dask_jobqueue.__version__ < "0.5.0":
raise ImportError
except ImportError:
raise ImportError("You need`dask-jobqueue >= 0.5.0` to use the `dask` adapter")
cluster = cluster_class(**dask_construct)
# Setup up adaption
# Workers are distributed down to the cores through the sub-divided processes
# Optimization may be needed
workers = settings.common.tasks_per_worker * settings.common.max_workers
if settings.cluster.adaptive == AdaptiveCluster.adaptive:
cluster.adapt(minimum=0, maximum=workers, interval="10s")
else:
cluster.scale(workers)
queue_client = Client(cluster)
elif settings.common.adapter == "parsl":
scheduler_opts = settings.cluster.scheduler_options
if not settings.cluster.node_exclusivity:
raise ValueError(
"For now, QCFractal can only be run with Parsl in node exclusivity. This will be relaxed "
"in a future release of Parsl and QCFractal"
)
# Import helpers
_provider_loaders = {
"slurm": "SlurmProvider",
"pbs": "TorqueProvider",
"moab": "TorqueProvider",
"sge": "GridEngineProvider",
"cobalt": "CobaltProvider",
"lsf": None,
}
if _provider_loaders[settings.cluster.scheduler] is None:
raise ValueError(f"Parsl does not know how to handle cluster of type {settings.cluster.scheduler}.")
# Headers
_provider_headers = {
"slurm": "#SBATCH",
"pbs": "#PBS",
"moab": "#PBS",
"sge": "#$$",
"lsf": None,
"cobalt": "#COBALT",
}
# Import the parsl things we need
try:
import parsl
from parsl.config import Config
from parsl.executors import HighThroughputExecutor
from parsl.addresses import address_by_hostname
provider_module = cli_utils.import_module(
"parsl.providers", package=_provider_loaders[settings.cluster.scheduler]
)
provider_class = getattr(provider_module, _provider_loaders[settings.cluster.scheduler])
provider_header = _provider_headers[settings.cluster.scheduler]
if parsl.__version__ < "0.9.0":
raise ImportError
except ImportError:
raise ImportError("You need `parsl >=0.9.0` to use the `parsl` adapter")
if _provider_loaders[settings.cluster.scheduler] == "moab":
logger.warning(
"Parsl uses its TorqueProvider for Moab clusters due to the scheduler similarities. "
"However, if you find a bug with it, please report to the Parsl and QCFractal developers so "
"it can be fixed on each respective end."
)
# Setup the providers
# Determine the maximum number of blocks
# TODO (wardlt): Math assumes that user does not set aside a compute node for the adapter
max_nodes = settings.common.max_workers * settings.common.nodes_per_task
if settings.common.nodes_per_job > max_nodes:
raise ValueError("Number of nodes per job is more than the maximum number of nodes used by manager")
if max_nodes % settings.common.nodes_per_job != 0:
raise ValueError(
"Maximum number of nodes (maximum number of workers times nodes per task) "
"needs to be a multiple of the number of nodes per job"
)
if settings.common.nodes_per_job % settings.common.nodes_per_task != 0:
raise ValueError("Number of nodes per job needs to be a multiple of the number of nodes per task")
max_blocks = max_nodes // settings.common.nodes_per_job
# Create one construct to quickly merge dicts with a final check
common_parsl_provider_construct = {
"init_blocks": 0, # Update this at a later time of Parsl
"max_blocks": max_blocks,
"walltime": settings.cluster.walltime,
"scheduler_options": f"{provider_header} " + f"\n{provider_header} ".join(scheduler_opts) + "\n",
"nodes_per_block": settings.common.nodes_per_job,
"worker_init": "\n".join(settings.cluster.task_startup_commands),
**settings.parsl.provider.dict(skip_defaults=True, exclude={"partition", "launcher"}),
}
if settings.cluster.scheduler.lower() == "slurm" and "cores_per_node" not in common_parsl_provider_construct:
common_parsl_provider_construct["cores_per_node"] = settings.common.cores_per_worker
# TODO: uncomment after Parsl#1416 is resolved
# if settings.cluster.scheduler.lower() == "slurm" and "mem_per_node" not in common_parsl_provider_construct:
# common_parsl_provider_construct["mem_per_node"] = settings.common.memory_per_worker
if settings.parsl.provider.launcher:
common_parsl_provider_construct["launcher"] = settings.parsl.provider.launcher.build_launcher()
if settings.cluster.scheduler == "slurm":
# The Parsl SLURM constructor has a strange set of arguments
provider = provider_class(
settings.parsl.provider.partition,
exclusive=settings.cluster.node_exclusivity,
**common_parsl_provider_construct,
)
else:
provider = provider_class(**common_parsl_provider_construct)
# The executor for Parsl is different for node parallel tasks and shared-memory tasks
if node_parallel_tasks:
# Tasks are launched from a single worker on the login node
# TODO (wardlt): Remove assumption that there is only one Parsl worker running all tasks
tasks_per_job = settings.common.nodes_per_job // settings.common.nodes_per_task
logger.info(f"Preparing a HTEx to use node-parallel tasks with {tasks_per_job} workers")
parsl_executor_construct = {
"label": "QCFractal_Parsl_{}_Executor".format(settings.cluster.scheduler.title()),
# Parsl will create one worker process per MPI task. Normally, Parsl prevents having
# more processes than cores. However, as each worker will spend most of its time
# waiting for the MPI task to complete, we can safely oversubscribe (e.g., more worker
# processes than cores), which requires setting "cores_per_worker" to <1
"cores_per_worker": 1e-6,
"max_workers": tasks_per_job,
"provider": provider,
"address": address_by_hostname(),
**settings.parsl.executor.dict(skip_defaults=True),
}
else:
parsl_executor_construct = {
"label": "QCFractal_Parsl_{}_Executor".format(settings.cluster.scheduler.title()),
"cores_per_worker": cores_per_task,
"max_workers": settings.common.tasks_per_worker,
"provider": provider,
"address": address_by_hostname(),
**settings.parsl.executor.dict(skip_defaults=True),
}
queue_client = Config(
retries=settings.common.retries, executors=[HighThroughputExecutor(**parsl_executor_construct)]
)
else:
raise KeyError(
"Unknown adapter type '{}', available options: {}.\n"
"This code should also be unreachable with pydantic Validation, so if "
"you see this message, please report it to the QCFractal GitHub".format(
settings.common.adapter, [getattr(AdapterEnum, v).value for v in AdapterEnum]
)
)
# Build out the manager itself
# Compute max tasks
max_concurrent_tasks = settings.common.tasks_per_worker * settings.common.max_workers
if settings.manager.max_queued_tasks is None:
# Tasks * jobs * buffer + 1
max_queued_tasks = ceil(max_concurrent_tasks * 2.00) + 1
else:
max_queued_tasks = settings.manager.max_queued_tasks
# The queue manager is configured differently for node-parallel and single-node tasks
manager = qcfractal.queue.QueueManager(
client,
queue_client,
max_tasks=max_queued_tasks,
queue_tag=settings.manager.queue_tag,
manager_name=settings.manager.manager_name,
update_frequency=settings.manager.update_frequency,
cores_per_task=cores_per_task,
memory_per_task=memory_per_task,
nodes_per_task=settings.common.nodes_per_task,
scratch_directory=settings.common.scratch_directory,
retries=settings.common.retries,
verbose=settings.common.verbose,
cores_per_rank=settings.common.cores_per_rank,
configuration=settings,
)
# Set stats correctly since we buffer the max tasks a bit
manager.statistics.max_concurrent_tasks = max_concurrent_tasks
# Add exit callbacks
for cb in exit_callbacks:
manager.add_exit_callback(cb[0], *cb[1], **cb[2])
# Either startup the manager or run until complete
if settings.manager.test:
success = manager.test(settings.manager.ntests)
if success is False:
raise ValueError("Testing was not successful, failing.")
else:
for signame in {"SIGHUP", "SIGINT", "SIGTERM"}:
def stop(*args, **kwargs):
manager.stop(signame)
raise KeyboardInterrupt()
signal.signal(getattr(signal, signame), stop)
# Blocks until signal
try:
manager.start()
except KeyboardInterrupt:
pass
#outside running on UI, snippet, partition, address_by_interface
""" Each job submitted to the scheduler will request 2 nodes for 10 minutes.
"""
config = Config(
executors=[
HighThroughputExecutor(
label="sd_htex",
address=address_by_interface('ib0'),
max_workers=1, # Set number of workers per node
provider=SlurmProvider(
cmd_timeout=60, # Add extra time for slow scheduler responses
channel=LocalChannel(),
nodes_per_block=2,
init_blocks=1,
min_blocks=1,
max_blocks=1,
partition='normal', # Replace with partition name
scheduler_options='#SBATCH -A <YOUR_ALLOCATION>', # Enter scheduler_options if needed
# Command to be run before starting a worker, such as:
# 'module load Anaconda; source activate parsl_env'.
worker_init='',
# Ideally we set the walltime to the longest supported walltime.
walltime='00:10:00',
launcher=SrunLauncher(),
),
)
],
)
from parsl.launchers import SrunLauncher
from parsl.executors import HighThroughputExecutor
from parsl.tests.utils import get_rundir
# If you are a developer running tests, make sure to update parsl/tests/configs/user_opts.py
# If you are a user copying-and-pasting this as an example, make sure to either
# 1) create a local `user_opts.py`, or
# 2) delete the user_opts import below and replace all appearances of `user_opts` with the literal value
# (i.e., user_opts['swan']['username'] -> 'your_username')
from .user_opts import user_opts
config = Config(executors=[
HighThroughputExecutor(
label='midway_htex_multinode',
provider=SlurmProvider(
'westmere',
channel=SSHChannel(hostname='swift.rcc.uchicago.edu',
username=user_opts['midway']['username'],
script_dir=user_opts['midway']['script_dir']),
launcher=SrunLauncher(),
scheduler_options=user_opts['midway']['scheduler_options'],
worker_init=user_opts['midway']['worker_init'],
walltime="00:05:00",
init_blocks=1,
max_blocks=1,
nodes_per_block=2,
),
)
],
run_dir=get_rundir())
from parsl.config import Config
from parsl.executors import HighThroughputExecutor
from parsl.launchers import MpiRunLauncher
from parsl.providers import CobaltProvider
config = Config(
executors=[
HighThroughputExecutor(
label="cooley_htex",
worker_debug=False,
cores_per_worker=1,
provider=CobaltProvider(
queue='debug',
account='YOUR_ACCOUNT', # project name to submit the job
launcher=MpiRunLauncher(),
scheduler_options=
'', # string to prepend to #COBALT blocks in the submit script to the scheduler
worker_init=
'', # command to run before starting a worker, such as 'source activate env'
init_blocks=1,
max_blocks=1,
min_blocks=1,
nodes_per_block=4,
cmd_timeout=60,
walltime='00:10:00',
),
)
], )
HighThroughputExecutor(
label='comet_htex',
worker_debug=True,
address='js-17-185.jetstream-cloud.org',
max_workers=24,
#workers_per_node = 24, # Getting error for unexpexted argument
cores_per_worker=1,
worker_logdir_root='/home/aymen/parsl_scripts',
interchange_address=
'comet-ln2.sdsc.edu', #'js-17-185.jetstream-cloud.org','comet-ln2.sdsc.edu',
interchange_port_range=(50000, 50400),
worker_port_range=(50500, 51000),
provider=SlurmProvider(
'debug',
channel=SSHChannel(
hostname='comet-ln2.sdsc.edu',
username=
'******', # Please replace USERNAME with your username
password='******',
script_dir=
'/home/aymen/parsl_scripts', # Please replace USERNAME with your username
),
#launcher=SrunLauncher(),
scheduler_options='', # Input your scheduler_options if needed
worker_init=
'source ve/parsl-env/bin/activate', # Input your worker_init if needed
#worker_init='source activate /oasis/projects/nsf/unc100/aymen/anaconda3/envs/conda-parsl',
#partition = "debug",
walltime="00:10:00",
init_blocks=1,
max_blocks=1,
#tasks_per_node = 24, # Getting error for unexpexted argument
nodes_per_block=1,
#cores_per_node=24, # Getting error for unexpexted argument
parallelism=24,
),
#working_dir="/home/aymen/parsl_scripts",
)
'''
nproc = 36
sched_opts = '''
#SBATCH --cpus-per-task=%d
''' % (nproc)
slurm_htex = Config(
executors=[
HighThroughputExecutor(
label="coffea_parsl_slurm",
address=address_by_hostname(),
prefetch_capacity=0,
max_workers=nproc,
provider=SlurmProvider(
channel=LocalChannel(script_dir='parsl_slurm'),
launcher=SrunLauncher(),
max_blocks=(args.ncpu)+5,
init_blocks=args.ncpu,
partition='all',
scheduler_options=sched_opts, # Enter scheduler_options if needed
worker_init=wrk_init, # Enter worker_init if needed
walltime='00:120:00'
),
)
],
retries=20,
)
dfk = parsl.load(slurm_htex)
else:
config = Config(executors=[ThreadPoolExecutor(max_threads=args.ncpu)])
parsl.load(config)
from parsl.config import Config
from parsl.providers import LocalProvider
from parsl.channels import SSHChannel
from parsl.addresses import address_by_hostname
from parsl.executors import HighThroughputExecutor
hostnames = ['host-1', 'host-2']
config = Config(
executors=[
HighThroughputExecutor(
label='htex_{}'.format(h),
worker_debug=False,
address=address_by_hostname(),
provider=LocalProvider(
# The username on the machines depend on the distribution
# used, for eg. on Ubuntu, username is 'ubuntu'
channel=SSHChannel(hostname=h, username='******'),
move_files=False, # set to True if there is no shared filesystem
nodes_per_block=1,
init_blocks=1,
min_blocks=1,
max_blocks=1,
# Command to be run before starting a worker, such as:
# 'module load Anaconda; source activate parsl_env'.
worker_init='',
),
) for h in hostnames
],
strategy=None)