Пример #1
0
def fresh_config():
    config = Config(
        executors=[
            HighThroughputExecutor(
                label="bw_htex",
                cores_per_worker=1,
                worker_debug=False,
                max_workers=1,
                address=address_by_hostname(),
                provider=TorqueProvider(
                    queue='normal',
                    launcher=AprunLauncher(overrides="-b -- bwpy-environ --"),
                    # string to prepend to #SBATCH blocks in the submit
                    # script to the scheduler eg: '#SBATCH --constraint=knl,quad,cache'
                    scheduler_options='',
                    # Command to be run before starting a worker, such as:
                    # 'module load Anaconda; source activate parsl_env'.
                    worker_init=user_opts['bluewaters']['worker_init'],
                    init_blocks=1,
                    max_blocks=1,
                    min_blocks=1,
                    nodes_per_block=2,
                    walltime='00:30:00',
                    cmd_timeout=120,
                ),
            )
        ], )
    return config
Пример #2
0
from parsl.config import Config
from parsl.executors import HighThroughputExecutor

# If you are a developer running tests, make sure to update parsl/tests/configs/user_opts.py
# If you are a user copying-and-pasting this as an example, make sure to either
#       1) create a local `user_opts.py`, or
#       2) delete the user_opts import below and replace all appearances of `user_opts` with the literal value
#          (i.e., user_opts['swan']['username'] -> 'your_username')
from .user_opts import user_opts

config = Config(
    executors=[
        HighThroughputExecutor(
            label='swan_htex',
            provider=TorqueProvider(
                channel=SSHChannel(
                    hostname='swan.cray.com',
                    username=user_opts['swan']['username'],
                    script_dir=user_opts['swan']['script_dir'],
                ),
                nodes_per_block=1,
                init_blocks=1,
                max_blocks=1,
                launcher=AprunLauncher(),
                scheduler_options=user_opts['swan']['scheduler_options'],
                worker_init=user_opts['swan']['worker_init'],
            ),
        )
    ]
)
Пример #3
0
# This is an example config, make sure to
#        replace the specific values below with the literal values
#          (e.g., 'USERNAME' -> 'your_username')

config = Config(
    executors=[
        IPyParallelExecutor(
            label='swan_ipp',
            workers_per_node=2,
            provider=TorqueProvider(
                channel=SSHChannel(
                    hostname='swan.cray.com',
                    username=
                    '******',  # Please replace USERNAME with your username
                    script_dir=
                    '/home/users/USERNAME/parsl_scripts',  # Please replace USERNAME with your username
                ),
                nodes_per_block=2,
                init_blocks=1,
                max_blocks=1,
                launcher=AprunLauncher(),
                scheduler_options='',  # Input your scheduler_options if needed
                worker_init='',  # Input your worker_init if needed
            ),
            controller=Controller(
                public_ip='PUBLIC_IP'
            ),  # Please replace PUBLIC_IP with your public ip
        )
    ], )
Пример #4
0
from parsl.config import Config
from parsl.executors import HighThroughputExecutor
from parsl.launchers import SingleNodeLauncher
from parsl.providers import TorqueProvider
from parsl.addresses import address_by_hostname
import os

config = Config(executors=[
    HighThroughputExecutor(
        cores_per_worker=1,
        mem_per_worker=4,
        max_workers=12,
        worker_debug=True,
        address=address_by_hostname(),
        provider=TorqueProvider(
            launcher=SingleNodeLauncher(),
            worker_init=
            ("module load gcc/6.2.0 miniconda3/4.7.10; "
             "source activate parsl_env; export PYTHONPATH='{}:{{PYTHONPATH}}'"
             ).format(os.getcwd()),
            init_blocks=1,
            max_blocks=10,
            min_blocks=0,
            nodes_per_block=1,
            walltime='99:00:00',
            scheduler_options='#PBS -l mem=48gb,nodes=1:ppn=12'),
    ),
],
                checkpoint_mode='task_exit')
Пример #5
0
from parsl.executors.ipp import IPyParallelExecutor
from parsl.tests.utils import get_rundir

# If you are a developer running tests, make sure to update parsl/tests/configs/user_opts.py
# If you are a user copying-and-pasting this as an example, make sure to either
#       1) create a local `user_opts.py`, or
#       2) delete the user_opts import below and replace all appearances of `user_opts` with the literal value
#          (i.e., user_opts['swan']['username'] -> 'your_username')
from .user_opts import user_opts

config = Config(executors=[
    IPyParallelExecutor(
        label='beagle_multinode_mpi',
        workers_per_node=1,
        provider=TorqueProvider(
            'debug',
            channel=SSHChannel(
                hostname='login4.beagle.ci.uchicago.edu',
                username=user_opts['beagle']['username'],
                script_dir="/lustre/beagle2/{}/parsl_scripts".format(
                    user_opts['beagle']['username'])),
            nodes_per_block=1,
            init_blocks=1,
            max_blocks=1,
            launcher=AprunLauncher(),
            scheduler_options=user_opts['beagle']['scheduler_options'],
            worker_init=user_opts['beagle']['worker_init'],
        ))
],
                run_dir=get_rundir())
Пример #6
0
from parsl.launchers import AprunLauncher
from parsl.providers import TorqueProvider

from parsl.config import Config
from parsl.executors import HighThroughputExecutor
from parsl.addresses import address_by_hostname

from parsl.data_provider.scheme import GlobusScheme

config = Config(executors=[
    HighThroughputExecutor(
        label="bluewaters_htex",
        worker_debug=True,
        address="<LOGIN_NODE>",
        provider=TorqueProvider(
            channel=LocalChannel(),
            init_blocks=1,
            max_blocks=1,
            min_blocks=1,
            nodes_per_block=1,
            launcher=AprunLauncher(overrides="-b -- bwpy-environ --"),
            scheduler_options='''#PBS -l nodes=1:ppn=32
#PBS -q debug''',
            worker_init='''module load bwpy''',
            walltime='00:30:00'),
        storage_access=[
            GlobusScheme(endpoint_uuid="d59900ef-6d04-11e5-ba46-22000b92c6ec",
                         endpoint_path="/",
                         local_path="/")
        ])
], )
Пример #7
0
from parsl.config import Config
from parsl.executors.ipp import IPyParallelExecutor

# This is an example config, make sure to
#        replace the specific values below with the literal values
#          (e.g., 'USERNAME' -> 'your_username')

config = Config(
    executors=[
        IPyParallelExecutor(
            label='beagle_multinode_mpi',
            provider=TorqueProvider(
                'debug',
                channel=SSHChannel(
                    hostname='login4.beagle.ci.uchicago.edu',
                    username=
                    '******',  # Please replace USERNAME with your username
                    script_dir=
                    '/lustre/beagle2/USERNAME/parsl_scripts',  # Please replace USERNAME with your username
                ),
                nodes_per_block=1,
                tasks_per_node=1,
                init_blocks=1,
                max_blocks=1,
                launcher=AprunLauncher(),
                scheduler_options='',  # Input your scheduler_options if needed
                worker_init='',  # Input your worker_init if needed
            ))
    ], )
Пример #8
0
from parsl.providers import TorqueProvider
from parsl.channels import LocalChannel
from parsl.config import Config
from parsl.executors import HighThroughputExecutor
from parsl.launchers import SingleNodeLauncher

config = Config(executors=[
    HighThroughputExecutor(
        label="test",
        max_workers=1,
        provider=TorqueProvider(
            launcher=SingleNodeLauncher(),
            account="mfknie",
            walltime="10:00:00",
            scheduler_options="#PBS -l mem=8gb -l nodes=1:ppn=2 -N test",
            worker_init="\n".join(("module load gcc/6.2.0",
                                   "module load pigz")),
            # "module load miniconda3/4.7.10",
            # "module load R/4.0.3",
            # "source activate /gpfs/data/gao-lab/software/local_ancest")),
            nodes_per_block=1,
            init_blocks=1,
            min_blocks=1,
            max_blocks=1,
            # parallelism=1
        ))
])
Пример #9
0
config = Config(
    executors=[
        HighThroughputExecutor(
            max_workers_per_node=1,
            worker_debug=False,
            address=address_by_hostname(),
            provider=TorqueProvider(
                queue='normal',
                launcher=AprunLauncher(overrides="-b -- bwpy-environ --"),

                # string to prepend to #SBATCH blocks in the submit
                scheduler_options=user_opts['bluewaters']['scheduler_options'],

                # Command to be run before starting a worker, such as:
                # 'module load bwpy; source activate funcx env'.
                worker_init=user_opts['bluewaters']['worker_init'],

                # Scale between 0-1 blocks with 2 nodes per block
                nodes_per_block=2,
                init_blocks=0,
                min_blocks=0,
                max_blocks=1,

                # Hold blocks for 30 minutes
                walltime='00:30:00'),
        )
    ], )

# fmt: on
Пример #10
0
from parsl.config import Config
from parsl.executors import HighThroughputExecutor
from parsl.addresses import address_by_hostname
from parsl.launchers import SingleNodeLauncher
from parsl.providers import TorqueProvider


config = Config(
    executors=[
        HighThroughputExecutor(
            # cores_per_worker=2,
            mem_per_worker=600,
            worker_debug=True,
            address=address_by_hostname(),
            provider=TorqueProvider(
                launcher=SingleNodeLauncher(),
                worker_init="export PYTHONPATH='{}:{{PYTHONPATH}}'".format(os.getcwd()),
                init_blocks=1,
                max_blocks=10,
                min_blocks=1,
                nodes_per_block=1,
                walltime='00:320:00',
                scheduler_options='#PBS -l mem=600gb'
            ),
        )
    ],
    checkpoint_mode='task_exit',
    retries=3
)
Пример #11
0
from parsl.config import Config
from parsl.executors import HighThroughputExecutor
from parsl.launchers import AprunLauncher
from parsl.providers import TorqueProvider

config = Config(
    executors=[
        HighThroughputExecutor(
            label="bw_htex",
            cores_per_worker=1,
            worker_debug=False,
            provider=TorqueProvider(
                queue='normal',
                launcher=AprunLauncher(overrides="-b -- bwpy-environ --"),
                scheduler_options=
                '',  # string to prepend to #SBATCH blocks in the submit script to the scheduler
                worker_init=
                '',  # command to run before starting a worker, such as 'source activate env'
                init_blocks=1,
                max_blocks=1,
                min_blocks=1,
                nodes_per_block=2,
                walltime='00:10:00'),
        )
    ], )