from parsl.executors import HighThroughputExecutor remotes = [ 'midway2-login2.rcc.uchicago.edu', 'midway2-login1.rcc.uchicago.edu' ] config = Config(executors=[ HighThroughputExecutor( label='AdHoc', max_workers=2, worker_logdir_root="/scratch/midway2/yadunand/parsl_scripts", provider=AdHocProvider( worker_init="source /scratch/midway2/yadunand/parsl_env_setup.sh", channels=[ SSHChannel( hostname=m, username="******", script_dir="/scratch/midway2/yadunand/parsl_cluster") for m in remotes ])) ]) @python_app def platform(sleep=2, stdout=None): import platform import time time.sleep(sleep) return platform.uname()
from parsl.providers import AdHocProvider from parsl.channels import SSHChannel from parsl.executors import HighThroughputExecutor from parsl.config import Config from parsl.tests.configs.user_opts import user_opts config = Config(executors=[ HighThroughputExecutor( label='remote_htex', cores_per_worker=1, worker_debug=False, address=user_opts['public_ip'], provider=AdHocProvider( move_files=False, parallelism=1, worker_init=user_opts['adhoc']['worker_init'], channels=[ SSHChannel( hostname=m, username=user_opts['adhoc']['username'], script_dir=user_opts['adhoc']['script_dir'], ) for m in user_opts['adhoc']['remote_hostnames'] ])) ], )
from parsl.executors import HighThroughputExecutor from parsl.config import Config user_opts = {'adhoc': {'username': '******', 'script_dir': 'YOUR_SCRIPT_DIR', 'remote_hostnames': ['REMOTE_HOST_URL_1', 'REMOTE_HOST_URL_2'] } } config = Config( executors=[ HighThroughputExecutor( label='remote_htex', max_workers=2, worker_logdir_root=user_opts['adhoc']['script_dir'], provider=AdHocProvider( # Command to be run before starting a worker, such as: # 'module load Anaconda; source activate parsl_env'. worker_init='', channels=[SSHChannel(hostname=m, username=user_opts['adhoc']['username'], script_dir=user_opts['adhoc']['script_dir'], ) for m in user_opts['adhoc']['remote_hostnames']] ) ) ], # AdHoc Clusters should not be setup with scaling strategy. strategy=None, )
from parsl.config import Config from parsl.channels import LocalChannel from parsl.executors import HighThroughputExecutor from parsl.providers import AdHocProvider config = Config(executors=[ HighThroughputExecutor(label='AdHoc', provider=AdHocProvider( channels=[LocalChannel(), LocalChannel()])) ])