def _get_executor(container):
"""
Get the Parsl executor from the container
Returns
-----------
executor
"""
executor = HighThroughputExecutor(
label=container["container_uuid"],
cores_per_worker=1,
max_workers=1,
poll_period=10,
# launch_cmd="ls; sleep 3600",
worker_logdir_root="runinfo",
# worker_debug=True,
address=address_by_route(),
provider=KubernetesProvider(
namespace="dlhub-privileged",
image=container["location"],
nodes_per_block=1,
init_blocks=1,
max_blocks=1,
parallelism=1,
worker_init="""pip install git+https://github.com/Parsl/parsl;
pip install git+https://github.com/funcx-faas/funcX;
export PYTHONPATH=$PYTHONPATH:/home/ubuntu:/app""",
# security=None,
secret="ryan-kube-secret",
pod_name=container["name"]
.replace(".", "-")
.replace("_", "-")
.replace("/", "-")
.lower(),
# secret="minikube-aws-ecr",
# user_id=32781,
# group_id=10253,
# run_as_non_root=True
),
)
return [executor]
def fresh_config():
worker_init_file = user_opts['petrelkube']['worker_init']
with open(os.path.expanduser(worker_init_file)) as f:
lines = f.readlines()
worker_init = ';'.join([line.strip() for line in lines])
config = Config(executors=[
HighThroughputExecutor(
label='kube-htex',
cores_per_worker=1,
max_workers=1,
worker_logdir_root='.',
# Address for the pod worker to connect back
address=address_by_route(),
provider=KubernetesProvider(
namespace="dlhub-privileged",
# Docker image url to use for pods
image='python:3.7',
# Command to be run upon pod start, such as:
# 'module load Anaconda; source activate parsl_env'.
# or 'pip install parsl'
worker_init=worker_init,
# The secret key to download the image
secret="ryan-kube-secret",
# Should follow the Kubernetes naming rules
pod_name='parsl-site-test-pod',
nodes_per_block=1,
init_blocks=1,
# Maximum number of pods to scale up
max_blocks=2,
),
),
])
return config
import parsl
from parsl.app.app import bash_app
from parsl.config import Config
from parsl.providers import GridEngineProvider
from parsl.executors import HighThroughputExecutor
from parsl.addresses import address_by_route
from data_generation import generate_data
# Parsl config for use on LC's campus cluster
# You should be able to change this configuration and reproduce the same results
config = Config(executors=[
HighThroughputExecutor(worker_debug=True,
cores_per_worker=16,
address=address_by_route(),
provider=GridEngineProvider(
walltime='10000:00:00',
nodes_per_block=1,
init_blocks=1,
max_blocks=3,
scheduler_options="#$ -pe smp 48"),
label="workers")
], )
# Enable parsl logging if you want, but it prints out a lot of (useful) info
parsl.set_stream_logger()
parsl.load(config)
@bash_app
from parsl.config import Config
from parsl.executors import HighThroughputExecutor
from parsl.providers import KubernetesProvider
from parsl.addresses import address_by_route
config = Config(
executors=[
HighThroughputExecutor(
label='kube-htex',
cores_per_worker=1,
max_workers=1,
worker_logdir_root='runinfo',
address=address_by_route(), # Address for the pod worker to connect back
provider=KubernetesProvider(
namespace="default",
image='CONTAINER_LOCATION', # Specify where to download the image
nodes_per_block=1,
init_blocks=1,
max_blocks=10, # Maximum number of pods to scale up
worker_init="""pip install parsl""", # install Parsl when the pod starts
secret="kube-secret", # The secret key to download the image
pod_name='test-pod', # Should follow the Kubernetes naming rules
),
),
]
)