def create_job_config(v230=False): coiled.create_job_configuration( name=V230_JOB if v230 else NIGHTLY_JOB, software="gjoseph92/scheduler-benchmark-230" if v230 else "gjoseph92/scheduler-benchmark", cpu=1, memory="4 GiB", command=["python", "nightly-run.py", "coiled"], # TODO how to securely add a github api token so we can run the script instead, and upload the gist? # then this will actually work as a fire-and-forget job. files=[ "nightly-benchmark/nightly-run.py", "nightly-benchmark/assertions.py", "run-coiled-benchmark-job.sh" ], )
def create_notebook(v230=False): coiled.create_job_configuration( name=(V230_JOB if v230 else NIGHTLY_JOB) + "-nb", software="gjoseph92/scheduler-benchmark-230" if v230 else "gjoseph92/scheduler-benchmark", cpu=1, memory="4 GiB", command=[ "jupyter", "lab", "--allow-root", "--ip=0.0.0.0", "--no-browser" ], # TODO why does this start in `sh` and not `bash`? ports=[8888], files=[ "nightly-benchmark/nightly-run.py", "nightly-benchmark/assertions.py", "run-coiled-benchmark-job.sh" ], )
import coiled # Create cluster software environment software_name = "examples/optuna-xgboost" coiled.create_software_environment( name=software_name, conda="environment.yaml", ) # Create notebook job software environment software_notebook_name = software_name + "-notebook" coiled.create_software_environment( name=software_notebook_name, container="coiled/notebook:latest", conda="environment.yaml", ) coiled.create_job_configuration( name="examples/optuna", software=software_notebook_name, command=[ "/bin/bash", "run.sh", ], files=["optuna-xgboost.ipynb", "workspace.json", "run.sh"], ports=[8888], description="Hyperparameter optimization with Optuna", )
software_name = "examples/hyperband-optimization" coiled.create_software_environment( name=software_name, conda=conda, ) # Create notebook job software environment software_notebook_name = software_name + "-notebook" coiled.create_software_environment( name=software_notebook_name, container="coiled/notebook:latest", conda=conda, ) coiled.create_job_configuration( name="examples/hyperband-optimization", software=software_notebook_name, command=[ "/bin/bash", "run.sh", ], files=[ "hyperband-optimization.ipynb", "torch_model.py", "workspace.json", "run.sh", ], ports=[8888], description="Tune a PyTorch model with Hyperband cross-validation", )
#!/usr/bin/env python3 import coiled if __name__ == "__main__": coiled.create_job_configuration( name="profiling", software="gjoseph92/profiling", cpu=2, memory="4 GiB", command=[ "/bin/bash", "-c", "SHELL=/bin/bash jupyter lab --allow-root --ip=0.0.0.0 --no-browser", ], ports=[8888], files=["dask_profiling_coiled/run_profile.py"], )
import coiled software_name = "examples/quickstart-notebook" coiled.create_software_environment(name=software_name, container="coiled/notebook:latest", conda="environment.yaml") coiled.create_job_configuration( name="examples/quickstart", software=software_name, command=[ "/bin/bash", "run.sh", ], files=["quickstart.ipynb", "workspace.json", "run.sh"], ports=[8888], description="Quickly launch a Dask cluster on the cloud with Coiled", )
import coiled # Create cluster software environment software_name = "examples/scaling-xgboost" coiled.create_software_environment( name=software_name, conda="environment.yaml", ) # Create notebook job software environment software_notebook_name = software_name + "-notebook" coiled.create_software_environment( name=software_notebook_name, container="coiled/notebook:latest", conda="environment.yaml", ) coiled.create_job_configuration( name="examples/scaling-xgboost", software=software_notebook_name, command=[ "/bin/bash", "run.sh", ], files=["scaling-xgboost.ipynb", "workspace.json", "run.sh"], ports=[8888], description="Perform distributed training of an XGBoost classifier", )
"dask", "dask-ml", "dask>=2.23.0", "fastparquet", "matplotlib", "pandas>=1.1.0", "python-snappy", "s3fs", "scikit-learn", "xgboost>=1.3.0", "optuna<2.4.0", "numpy", "xgboost", "joblib", ] }, pip=["dask-optuna"], ) coiled.create_job_configuration( name="blog-notebooks/xgboost-on-coiled", software=software_name, command=[ "/bin/bash", "run.sh", ], files=["xgboost-mortgage.ipynb", "workspace.json", "run.sh"], ports=[8888], description="Train XGBoost on a large dataset with Dask on Coiled", )
"dask", "dask-ml", "dask>=2.23.0", "fastparquet", "matplotlib", "pandas>=1.1.0", "python-snappy", "s3fs", "scikit-learn", "xgboost>=1.3.0", "optuna<2.4.0", "numpy", "xgboost", "joblib", ] }, pip=["dask-optuna"], ) coiled.create_job_configuration( name="blog-notebooks/optuna-xgboost", software=software_name, command=[ "/bin/bash", "run.sh", ], files=["hyperparameter-tuning.ipynb", "workspace.json", "run.sh"], ports=[8888], description="XGBoost hyperparameter tuning with Optuna and Dask on Coiled", )
import coiled # Create cluster software environment software_name = "examples/dask-sql" coiled.create_software_environment( name=software_name, conda="cluster-env.yaml", ) # Create notebook job software environment software_notebook_name = software_name + "-notebook" # Add Dask-SQL and matplotlib to notebook software environment coiled.create_software_environment( name=software_notebook_name, container="coiled/notebook:latest", conda="notebook-env.yaml", ) coiled.create_job_configuration( name="examples/dask-sql", software=software_notebook_name, command=[ "/bin/bash", "run.sh", ], files=["dask-sql.ipynb", "workspace.json", "run.sh"], ports=[8888], description="Query and transform Dask DataFrames using SQL", )
conda = { "channels": ["conda-forge"], "dependencies": [ "python=3.8", "traitlets=5.0.4", "dask=2021.3.0", "coiled=0.0.37", ], } software_name = "examples/jupyterlab-notebook" coiled.create_software_environment( name=software_name, container="coiled/notebook:latest", conda=conda, ) coiled.create_job_configuration( name="examples/jupyterlab", software=software_name, command=[ "/bin/bash", "run.sh", ], files=[ "jupyterlab.ipynb", "workspace.json", "run.sh", "dask-extension.png" ], ports=[8888], description="See how Coiled intergrates with JupyterLab", )