from parsl.config import Config
from parsl.executors import WorkQueueExecutor
import uuid
config = Config(executors=[
WorkQueueExecutor(
label="parsl-wq-example",
# If a project_name is given, then Work Queue will periodically
# report its status and performance back to the global WQ catalog,
# which can be viewed here: http://ccl.cse.nd.edu/software/workqueue/status
# To disable status reporting, comment out the project_name.
project_name="parsl-wq-" + str(uuid.uuid4()),
# The port number that Work Queue will listen on for connecting workers.
port=9123,
# A shared filesystem is not needed when using Work Queue.
shared_fs=False)
])
# Sample Parsl application uses Work Queue for execution.
import parsl
from parsl import python_app, bash_app
from parsl.executors import WorkQueueExecutor
import work_queue as wq
# Create the WQ executor to send tasks to workers.
# Note that the LocalProvider is used by default to start a worker.
config = parsl.config.Config(executors=[
WorkQueueExecutor(
label="wq_parsl_integration_test",
port=9123,
project_name="wq_parsl_integration_test",
shared_fs=False,
full_debug=True,
)
])
# This is the MapReduce example verbatim from the Parsl docs:
from parsl import python_app
parsl.load(config)
# Map function that returns double the input integer
@python_app
def app_double(x):
return x * 2
from parsl.config import Config
from parsl.executors import WorkQueueExecutor
config = Config(executors=[
WorkQueueExecutor(label="wqex_local",
port=50055,
project_name="WorkQueue Example",
shared_fs=True,
see_worker_output=True)
])
from parsl.config import Config
from parsl.executors import WorkQueueExecutor
config = Config(executors=[
WorkQueueExecutor(
port=50055,
project_name="WQexample",
see_worker_output=True,
source=True,
# init_command='source /home/yadu/src/wq_parsl/setup_parsl_env.sh;
# echo "Ran at $date" > /home/yadu/src/wq_parsl/parsl/tests/workqueue_tests/ran.log',
)
])
from parsl.config import Config
from parsl.executors import WorkQueueExecutor
from parsl.providers import LocalProvider
config = Config(executors=[
WorkQueueExecutor(
port=50055,
source=True,
provider=LocalProvider(),
# init_command='source /home/yadu/src/wq_parsl/setup_parsl_env.sh;
# echo "Ran at $date" > /home/yadu/src/wq_parsl/parsl/tests/workqueue_tests/ran.log',
)
])
from parsl.config import Config
from parsl.executors import WorkQueueExecutor
from parsl.data_provider.http import HTTPInTaskStaging
from parsl.data_provider.ftp import FTPInTaskStaging
from parsl.data_provider.file_noop import NoOpFileStaging
config = Config(executors=[
WorkQueueExecutor(port=9000,
storage_access=[
FTPInTaskStaging(),
HTTPInTaskStaging(),
NoOpFileStaging()
])
])
from parsl.config import Config
from parsl.executors import WorkQueueExecutor
from parsl.data_provider.http import HTTPInTaskStaging
from parsl.data_provider.ftp import FTPInTaskStaging
from parsl.data_provider.file_noop import NoOpFileStaging
config = Config(executors=[
WorkQueueExecutor(port=9000,
project_name="wq-testing",
storage_access=[
FTPInTaskStaging(),
HTTPInTaskStaging(),
NoOpFileStaging()
])
])
label='condor_hte',
provider=CondorProvider(project="condor_hte",
channel=LocalChannel(),
worker_init="export HOME=$PWD",
environment={"PATH": path_str}),
address=address_str,
working_dir=os.getcwd(),
),
])
#TODO: This port number needs to be configurable so that a script can specify
# the port here and when spinning up workers.
wq_config = Config(executors=[
WorkQueueExecutor(label='wq_exec',
project_name='wq_benchmark',
env={"PATH": path_str},
init_command="export HOME=$PWD",
port=9000)
])
def test_local_sequential(task_batch_sizes):
print("Local Sequential Workload")
test_parsl_config("local_sequential_workflow.png", local_config,
run_sequential_workflow, task_batch_sizes)
def test_local_parallel(task_batch_sizes):
print("Local Parallel Workload")
test_parsl_config("local_parallel_workflow.png", local_config,
run_local_parallel_workflow, task_batch_sizes)
from parsl.config import Config
from parsl.executors import WorkQueueExecutor
from parsl.data_provider.http import HTTPInTaskStaging
from parsl.data_provider.ftp import FTPInTaskStaging
from parsl.data_provider.file_noop import NoOpFileStaging
from parsl.providers import LocalProvider
config = Config(executors=[
WorkQueueExecutor(port=9000,
storage_access=[
FTPInTaskStaging(),
HTTPInTaskStaging(),
NoOpFileStaging()
],
provider=LocalProvider(
init_blocks=0, min_blocks=0, max_blocks=1))
])
from parsl.config import Config
from parsl.executors import WorkQueueExecutor
config = Config(executors=[
WorkQueueExecutor(
port=50055,
number_of_servers=5,
# init_command='source /home/yadu/src/wq_parsl/setup_parsl_env.sh;
# echo "Ran at $date" > /home/yadu/src/wq_parsl/parsl/tests/workqueue_tests/ran.log',
)
])
from parsl.config import Config
from parsl.executors import WorkQueueExecutor
config = Config(executors=[
WorkQueueExecutor(label="parsl_wq_example",
port=9123,
project_name="parsl_wq_example",
shared_fs=False)
])
from parsl.config import Config
from parsl.executors import WorkQueueExecutor
config = Config(executors=[WorkQueueExecutor(port=9000,
project_name="wq-testing")])