from parsl.executors import HighThroughputExecutor
remotes = [
'midway2-login2.rcc.uchicago.edu', 'midway2-login1.rcc.uchicago.edu'
]
config = Config(executors=[
HighThroughputExecutor(
label='AdHoc',
max_workers=2,
worker_logdir_root="/scratch/midway2/yadunand/parsl_scripts",
provider=AdHocProvider(
worker_init="source /scratch/midway2/yadunand/parsl_env_setup.sh",
channels=[
SSHChannel(
hostname=m,
username="******",
script_dir="/scratch/midway2/yadunand/parsl_cluster")
for m in remotes
]))
])
@python_app
def platform(sleep=2, stdout=None):
import platform
import time
time.sleep(sleep)
return platform.uname()
from parsl.providers import AdHocProvider
from parsl.channels import SSHChannel
from parsl.executors import HighThroughputExecutor
from parsl.config import Config
from parsl.tests.configs.user_opts import user_opts
config = Config(executors=[
HighThroughputExecutor(
label='remote_htex',
cores_per_worker=1,
worker_debug=False,
address=user_opts['public_ip'],
provider=AdHocProvider(
move_files=False,
parallelism=1,
worker_init=user_opts['adhoc']['worker_init'],
channels=[
SSHChannel(
hostname=m,
username=user_opts['adhoc']['username'],
script_dir=user_opts['adhoc']['script_dir'],
) for m in user_opts['adhoc']['remote_hostnames']
]))
], )
from parsl.executors import HighThroughputExecutor
from parsl.config import Config
user_opts = {'adhoc':
{'username': '******',
'script_dir': 'YOUR_SCRIPT_DIR',
'remote_hostnames': ['REMOTE_HOST_URL_1', 'REMOTE_HOST_URL_2']
}
}
config = Config(
executors=[
HighThroughputExecutor(
label='remote_htex',
max_workers=2,
worker_logdir_root=user_opts['adhoc']['script_dir'],
provider=AdHocProvider(
# Command to be run before starting a worker, such as:
# 'module load Anaconda; source activate parsl_env'.
worker_init='',
channels=[SSHChannel(hostname=m,
username=user_opts['adhoc']['username'],
script_dir=user_opts['adhoc']['script_dir'],
) for m in user_opts['adhoc']['remote_hostnames']]
)
)
],
# AdHoc Clusters should not be setup with scaling strategy.
strategy=None,
)
from parsl.config import Config
from parsl.channels import LocalChannel
from parsl.executors import HighThroughputExecutor
from parsl.providers import AdHocProvider
config = Config(executors=[
HighThroughputExecutor(label='AdHoc',
provider=AdHocProvider(
channels=[LocalChannel(),
LocalChannel()]))
])