def create_job_config(v230=False):
coiled.create_job_configuration(
name=V230_JOB if v230 else NIGHTLY_JOB,
software="gjoseph92/scheduler-benchmark-230"
if v230 else "gjoseph92/scheduler-benchmark",
cpu=1,
memory="4 GiB",
command=["python", "nightly-run.py", "coiled"],
# TODO how to securely add a github api token so we can run the script instead, and upload the gist?
# then this will actually work as a fire-and-forget job.
files=[
"nightly-benchmark/nightly-run.py",
"nightly-benchmark/assertions.py", "run-coiled-benchmark-job.sh"
],
)
def create_notebook(v230=False):
coiled.create_job_configuration(
name=(V230_JOB if v230 else NIGHTLY_JOB) + "-nb",
software="gjoseph92/scheduler-benchmark-230"
if v230 else "gjoseph92/scheduler-benchmark",
cpu=1,
memory="4 GiB",
command=[
"jupyter", "lab", "--allow-root", "--ip=0.0.0.0", "--no-browser"
],
# TODO why does this start in `sh` and not `bash`?
ports=[8888],
files=[
"nightly-benchmark/nightly-run.py",
"nightly-benchmark/assertions.py", "run-coiled-benchmark-job.sh"
],
)
import coiled
# Create cluster software environment
software_name = "examples/optuna-xgboost"
coiled.create_software_environment(
name=software_name,
conda="environment.yaml",
)
# Create notebook job software environment
software_notebook_name = software_name + "-notebook"
coiled.create_software_environment(
name=software_notebook_name,
container="coiled/notebook:latest",
conda="environment.yaml",
)
coiled.create_job_configuration(
name="examples/optuna",
software=software_notebook_name,
command=[
"/bin/bash",
"run.sh",
],
files=["optuna-xgboost.ipynb", "workspace.json", "run.sh"],
ports=[8888],
description="Hyperparameter optimization with Optuna",
)
software_name = "examples/hyperband-optimization"
coiled.create_software_environment(
name=software_name,
conda=conda,
)
# Create notebook job software environment
software_notebook_name = software_name + "-notebook"
coiled.create_software_environment(
name=software_notebook_name,
container="coiled/notebook:latest",
conda=conda,
)
coiled.create_job_configuration(
name="examples/hyperband-optimization",
software=software_notebook_name,
command=[
"/bin/bash",
"run.sh",
],
files=[
"hyperband-optimization.ipynb",
"torch_model.py",
"workspace.json",
"run.sh",
],
ports=[8888],
description="Tune a PyTorch model with Hyperband cross-validation",
)
#!/usr/bin/env python3
import coiled
if __name__ == "__main__":
coiled.create_job_configuration(
name="profiling",
software="gjoseph92/profiling",
cpu=2,
memory="4 GiB",
command=[
"/bin/bash",
"-c",
"SHELL=/bin/bash jupyter lab --allow-root --ip=0.0.0.0 --no-browser",
],
ports=[8888],
files=["dask_profiling_coiled/run_profile.py"],
)
import coiled
software_name = "examples/quickstart-notebook"
coiled.create_software_environment(name=software_name,
container="coiled/notebook:latest",
conda="environment.yaml")
coiled.create_job_configuration(
name="examples/quickstart",
software=software_name,
command=[
"/bin/bash",
"run.sh",
],
files=["quickstart.ipynb", "workspace.json", "run.sh"],
ports=[8888],
description="Quickly launch a Dask cluster on the cloud with Coiled",
)
import coiled
# Create cluster software environment
software_name = "examples/scaling-xgboost"
coiled.create_software_environment(
name=software_name,
conda="environment.yaml",
)
# Create notebook job software environment
software_notebook_name = software_name + "-notebook"
coiled.create_software_environment(
name=software_notebook_name,
container="coiled/notebook:latest",
conda="environment.yaml",
)
coiled.create_job_configuration(
name="examples/scaling-xgboost",
software=software_notebook_name,
command=[
"/bin/bash",
"run.sh",
],
files=["scaling-xgboost.ipynb", "workspace.json", "run.sh"],
ports=[8888],
description="Perform distributed training of an XGBoost classifier",
)
"dask",
"dask-ml",
"dask>=2.23.0",
"fastparquet",
"matplotlib",
"pandas>=1.1.0",
"python-snappy",
"s3fs",
"scikit-learn",
"xgboost>=1.3.0",
"optuna<2.4.0",
"numpy",
"xgboost",
"joblib",
]
},
pip=["dask-optuna"],
)
coiled.create_job_configuration(
name="blog-notebooks/xgboost-on-coiled",
software=software_name,
command=[
"/bin/bash",
"run.sh",
],
files=["xgboost-mortgage.ipynb", "workspace.json", "run.sh"],
ports=[8888],
description="Train XGBoost on a large dataset with Dask on Coiled",
)
"dask",
"dask-ml",
"dask>=2.23.0",
"fastparquet",
"matplotlib",
"pandas>=1.1.0",
"python-snappy",
"s3fs",
"scikit-learn",
"xgboost>=1.3.0",
"optuna<2.4.0",
"numpy",
"xgboost",
"joblib",
]
},
pip=["dask-optuna"],
)
coiled.create_job_configuration(
name="blog-notebooks/optuna-xgboost",
software=software_name,
command=[
"/bin/bash",
"run.sh",
],
files=["hyperparameter-tuning.ipynb", "workspace.json", "run.sh"],
ports=[8888],
description="XGBoost hyperparameter tuning with Optuna and Dask on Coiled",
)
import coiled
# Create cluster software environment
software_name = "examples/dask-sql"
coiled.create_software_environment(
name=software_name,
conda="cluster-env.yaml",
)
# Create notebook job software environment
software_notebook_name = software_name + "-notebook"
# Add Dask-SQL and matplotlib to notebook software environment
coiled.create_software_environment(
name=software_notebook_name,
container="coiled/notebook:latest",
conda="notebook-env.yaml",
)
coiled.create_job_configuration(
name="examples/dask-sql",
software=software_notebook_name,
command=[
"/bin/bash",
"run.sh",
],
files=["dask-sql.ipynb", "workspace.json", "run.sh"],
ports=[8888],
description="Query and transform Dask DataFrames using SQL",
)
conda = {
"channels": ["conda-forge"],
"dependencies": [
"python=3.8",
"traitlets=5.0.4",
"dask=2021.3.0",
"coiled=0.0.37",
],
}
software_name = "examples/jupyterlab-notebook"
coiled.create_software_environment(
name=software_name,
container="coiled/notebook:latest",
conda=conda,
)
coiled.create_job_configuration(
name="examples/jupyterlab",
software=software_name,
command=[
"/bin/bash",
"run.sh",
],
files=[
"jupyterlab.ipynb", "workspace.json", "run.sh", "dask-extension.png"
],
ports=[8888],
description="See how Coiled intergrates with JupyterLab",
)