from parsl.config import Config from parsl.executors import WorkQueueExecutor import uuid config = Config(executors=[ WorkQueueExecutor( label="parsl-wq-example", # If a project_name is given, then Work Queue will periodically # report its status and performance back to the global WQ catalog, # which can be viewed here: http://ccl.cse.nd.edu/software/workqueue/status # To disable status reporting, comment out the project_name. project_name="parsl-wq-" + str(uuid.uuid4()), # The port number that Work Queue will listen on for connecting workers. port=9123, # A shared filesystem is not needed when using Work Queue. shared_fs=False) ])
# Sample Parsl application uses Work Queue for execution. import parsl from parsl import python_app, bash_app from parsl.executors import WorkQueueExecutor import work_queue as wq # Create the WQ executor to send tasks to workers. # Note that the LocalProvider is used by default to start a worker. config = parsl.config.Config(executors=[ WorkQueueExecutor( label="wq_parsl_integration_test", port=9123, project_name="wq_parsl_integration_test", shared_fs=False, full_debug=True, ) ]) # This is the MapReduce example verbatim from the Parsl docs: from parsl import python_app parsl.load(config) # Map function that returns double the input integer @python_app def app_double(x): return x * 2
from parsl.config import Config from parsl.executors import WorkQueueExecutor config = Config(executors=[ WorkQueueExecutor(label="wqex_local", port=50055, project_name="WorkQueue Example", shared_fs=True, see_worker_output=True) ])
from parsl.config import Config from parsl.executors import WorkQueueExecutor config = Config(executors=[ WorkQueueExecutor( port=50055, project_name="WQexample", see_worker_output=True, source=True, # init_command='source /home/yadu/src/wq_parsl/setup_parsl_env.sh; # echo "Ran at $date" > /home/yadu/src/wq_parsl/parsl/tests/workqueue_tests/ran.log', ) ])
from parsl.config import Config from parsl.executors import WorkQueueExecutor from parsl.providers import LocalProvider config = Config(executors=[ WorkQueueExecutor( port=50055, source=True, provider=LocalProvider(), # init_command='source /home/yadu/src/wq_parsl/setup_parsl_env.sh; # echo "Ran at $date" > /home/yadu/src/wq_parsl/parsl/tests/workqueue_tests/ran.log', ) ])
from parsl.config import Config from parsl.executors import WorkQueueExecutor from parsl.data_provider.http import HTTPInTaskStaging from parsl.data_provider.ftp import FTPInTaskStaging from parsl.data_provider.file_noop import NoOpFileStaging config = Config(executors=[ WorkQueueExecutor(port=9000, storage_access=[ FTPInTaskStaging(), HTTPInTaskStaging(), NoOpFileStaging() ]) ])
from parsl.config import Config from parsl.executors import WorkQueueExecutor from parsl.data_provider.http import HTTPInTaskStaging from parsl.data_provider.ftp import FTPInTaskStaging from parsl.data_provider.file_noop import NoOpFileStaging config = Config(executors=[ WorkQueueExecutor(port=9000, project_name="wq-testing", storage_access=[ FTPInTaskStaging(), HTTPInTaskStaging(), NoOpFileStaging() ]) ])
label='condor_hte', provider=CondorProvider(project="condor_hte", channel=LocalChannel(), worker_init="export HOME=$PWD", environment={"PATH": path_str}), address=address_str, working_dir=os.getcwd(), ), ]) #TODO: This port number needs to be configurable so that a script can specify # the port here and when spinning up workers. wq_config = Config(executors=[ WorkQueueExecutor(label='wq_exec', project_name='wq_benchmark', env={"PATH": path_str}, init_command="export HOME=$PWD", port=9000) ]) def test_local_sequential(task_batch_sizes): print("Local Sequential Workload") test_parsl_config("local_sequential_workflow.png", local_config, run_sequential_workflow, task_batch_sizes) def test_local_parallel(task_batch_sizes): print("Local Parallel Workload") test_parsl_config("local_parallel_workflow.png", local_config, run_local_parallel_workflow, task_batch_sizes)
from parsl.config import Config from parsl.executors import WorkQueueExecutor from parsl.data_provider.http import HTTPInTaskStaging from parsl.data_provider.ftp import FTPInTaskStaging from parsl.data_provider.file_noop import NoOpFileStaging from parsl.providers import LocalProvider config = Config(executors=[ WorkQueueExecutor(port=9000, storage_access=[ FTPInTaskStaging(), HTTPInTaskStaging(), NoOpFileStaging() ], provider=LocalProvider( init_blocks=0, min_blocks=0, max_blocks=1)) ])
from parsl.config import Config from parsl.executors import WorkQueueExecutor config = Config(executors=[ WorkQueueExecutor( port=50055, number_of_servers=5, # init_command='source /home/yadu/src/wq_parsl/setup_parsl_env.sh; # echo "Ran at $date" > /home/yadu/src/wq_parsl/parsl/tests/workqueue_tests/ran.log', ) ])
from parsl.config import Config from parsl.executors import WorkQueueExecutor config = Config(executors=[ WorkQueueExecutor(label="parsl_wq_example", port=9123, project_name="parsl_wq_example", shared_fs=False) ])
from parsl.config import Config from parsl.executors import WorkQueueExecutor config = Config(executors=[WorkQueueExecutor(port=9000, project_name="wq-testing")])