Python HTTPInTaskStaging Examples

Example #1

def fresh_config():
    return Config(
        executors=[
            HighThroughputExecutor(
                label="htex_Local",
                working_dir=working_dir,
                storage_access=[FTPInTaskStaging(), HTTPInTaskStaging(), NoOpFileStaging()],
                worker_debug=True,
                cores_per_worker=1,
                heartbeat_period=2,
                heartbeat_threshold=5,
                poll_period=100,
                provider=LocalProvider(
                    channel=LocalChannel(),
                    init_blocks=0,
                    min_blocks=0,
                    max_blocks=5,
                    launcher=SingleNodeLauncher(),
                ),
            )
        ],
        strategy='simple',
        app_cache=True, checkpoint_mode='task_exit',
        retries=2,
        monitoring=MonitoringHub(
                        hub_address="localhost",
                        hub_port=55055,
                        monitoring_debug=False,
                        resource_monitoring_interval=1,
        )
    )

Example #2

File: local_threads_http_in_task.py Project: xinyixiang/parsl

from parsl.config import Config
from parsl.data_provider.file_noop import NoOpFileStaging
from parsl.data_provider.http import HTTPInTaskStaging
from parsl.executors.threads import ThreadPoolExecutor
from parsl.tests.utils import get_rundir

config = Config(
    executors=[
        ThreadPoolExecutor(
            label='local_threads_http_in_task',
            storage_access=[HTTPInTaskStaging(), NoOpFileStaging()]
        )
    ],
    run_dir=get_rundir()
)

Example #3

File: htex_local_rsync_staging.py Project: xinyixiang/parsl

from parsl.channels import LocalChannel
from parsl.launchers import SimpleLauncher

from parsl.data_provider.http import HTTPInTaskStaging
from parsl.data_provider.ftp import FTPInTaskStaging
from parsl.data_provider.rsync import RSyncStaging

from parsl.config import Config
from parsl.executors import HighThroughputExecutor

config = Config(
    executors=[
        HighThroughputExecutor(label="htex_Local",
                               worker_debug=True,
                               cores_per_worker=1,
                               working_dir="./rsync-workdir/",
                               provider=LocalProvider(
                                   channel=LocalChannel(),
                                   init_blocks=1,
                                   max_blocks=1,
                                   launcher=SimpleLauncher(),
                               ),
                               storage_access=[
                                   HTTPInTaskStaging(),
                                   FTPInTaskStaging(),
                                   RSyncStaging("fitzroy.cqx.ltd.uk")
                               ])
    ],
    strategy=None,
)

Example #4

File: azure_single_node.py Project: xinyixiang/parsl

vm_reference = {

    # these fields gave me key errors unless I specified them:
    "admin_username": user_opts['azure']['admin_username'],
    "password": user_opts['azure']['password'],
    "vm_size": user_opts['azure']['vm_size'],
    "disk_size_gb": user_opts['azure']['disk_size_gb'],
    "publisher": user_opts['azure']['publisher'],
    "offer": user_opts['azure']['offer'],
    "sku": user_opts['azure']['sku'],
    "version": user_opts['azure']['version']
}

provider = AzureProvider(
              vm_reference=vm_reference,
              key_file='azure_key_file.json',
            )

config = Config(
    executors=[
        HighThroughputExecutor(
            storage_access=[HTTPInTaskStaging(), FTPInTaskStaging(), RSyncStaging(getpass.getuser() + "@" + user_opts['public_ip'])],
            label='azure_single_node',
            address=user_opts['public_ip'],
            provider=provider
        )
    ],
    run_dir=get_rundir(),
)