def spin_up_cluster(n_workers): # set up the cluster before executing the run env_name = "cmip6_derived_cloud_datasets" ## use the pangeo containerfor the software env coiled.create_software_environment( name=env_name, conda= 'environment.yml', # this will take longer...but I guess thats ok for now? # couldnt get this to work yet # container='pangeo/pangeo-notebook:latest', # matches Pangeo Cloud AWS production cluster ) # Create a Dask cluster which uses # software environment cluster = coiled.Cluster( software=env_name, n_workers=n_workers, backend_options={"region": "us-west-2"}, shutdown_on_close=True, ) client = Client(cluster) print("Cluster Name:", cluster.name) print("Dashboard:", client.dashboard_link) print('\n\n\n----------------------------') return client, cluster
def __init__(self): config_file = os.path.join(os.getcwd(), "config", "config.yaml") if not os.path.isfile(config_file): raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), config_file) with open(config_file) as file: self.config_values = yaml.full_load(file) if not "cluster_name" in self.config_values: self.config_values["cluster_name"] = "chess-cluster" if not "software_environment_name" in self.config_values: self.config_values["software_environment_name"] = "chess-env" if not "n_workers" in self.config_values: self.config_values["n_workers"] = 50 if not "worker_cpu" in self.config_values: self.config_values["worker_cpu"] = 1 if not "worker_memory" in self.config_values: self.config_values["worker_memory"] = 8 if not "scheduler_memory" in self.config_values: self.config_values["scheduler_memory"] = 16 if not "scheduler_cpu" in self.config_values: self.config_values["scheduler_cpu"] = 4 if not "game_batch_size" in self.config_values: self.config_values["game_batch_size"] = 30 if not "timeout_per_game" in self.config_values: self.config_values["timeout_per_game"] = 60 if not "debug" in self.config_values: self.config_values["debug"] = False if self.config_values["use_local_cluster"]: cluster = LocalCluster(n_workers=self.config_values["n_workers"], threads_per_worker=1) else: coiled.create_software_environment( name=self.config_values["software_environment_name"], pip="requirements.txt") cluster = coiled.Cluster( name=self.config_values["cluster_name"], n_workers=self.config_values["n_workers"], worker_cpu=self.config_values["worker_cpu"], worker_memory=str(self.config_values["worker_memory"]) + "GiB", scheduler_memory=str(self.config_values["scheduler_memory"]) + "GiB", scheduler_cpu=self.config_values["scheduler_cpu"], software=self.config_values["software_environment_name"]) self.client = Client(cluster)
import coiled # Create cluster software environment software_name = "examples/optuna-xgboost" coiled.create_software_environment( name=software_name, conda="environment.yaml", ) # Create notebook job software environment software_notebook_name = software_name + "-notebook" coiled.create_software_environment( name=software_notebook_name, container="coiled/notebook:latest", conda="environment.yaml", ) coiled.create_job_configuration( name="examples/optuna", software=software_notebook_name, command=[ "/bin/bash", "run.sh", ], files=["optuna-xgboost.ipynb", "workspace.json", "run.sh"], ports=[8888], description="Hyperparameter optimization with Optuna", )
"python=3.8", "dask=2021.3.0", "coiled=0.0.37", "optuna", "numpy", "scikit-learn", "xgboost", "joblib", ], } # Create cluster software environment software_name = "examples/optuna-xgboost" coiled.create_software_environment( name=software_name, conda=conda, pip=["dask-optuna"], ) # Create notebook job software environment software_notebook_name = software_name + "-notebook" coiled.create_software_environment( name=software_notebook_name, container="coiled/notebook:latest", conda=conda, pip=["dask-optuna"], ) coiled.create_job_configuration( name="examples/optuna", software=software_notebook_name,
import coiled software_name = "blog-notebooks/xgboost-on-coiled" coiled.create_software_environment( name=software_name, container="coiled/notebook:latest", conda={ "channels": ["conda-forge"], "dependencies": [ "python=3.8", "coiled=0.0.36", "dask", "dask-ml", "dask>=2.23.0", "fastparquet", "matplotlib", "pandas>=1.1.0", "python-snappy", "s3fs", "scikit-learn", "xgboost>=1.3.0", "optuna<2.4.0", "numpy", "xgboost", "joblib", ] }, pip=["dask-optuna"], ) coiled.create_job_configuration( name="blog-notebooks/xgboost-on-coiled",
#!/usr/bin/env python3 import coiled import argparse coiled.create_software_environment( name="i2k-2020-mi2", conda="environment.yml", )
}, "pytorch": { "conda": "hyper-parameter-optimmization/environment.yml" }, } cluster_configurations = { "xgboost": { "worker_cpu": 4, "worker_memory": "8 GiB" }, "pangeo": { "worker_cpu": 4, "worker_memory": "8 GiB" }, "pytorch": { "worker_cpu": 4, "worker_memory": "16 GiB" }, } assert software_environments.keys() == cluster_configurations.keys() for name, spec in software_environments.items(): full_name = f"coiled-examples/{name}" print(f"Building {full_name}:") coiled.create_software_environment(name=full_name, **spec) coiled.create_cluster_configuration(name=full_name, software=full_name, **cluster_configurations[name])
#!/usr/bin/env python3 import coiled import argparse # conda=["pip", "nodejs", "graphviz", "compilers"], coiled.create_software_environment( name="sc20-pyhpc", conda="environment.yml", post_build=[ "jupyter labextension install @jupyter-widgets/jupyterlab-manager jupyter-matplotlib jupyterlab-datawidgets [email protected] dask-labextension [email protected] [email protected]", "jupyter serverextension enable dask_labextension" ], )
import coiled software_name = "examples/quickstart-notebook" coiled.create_software_environment( name=software_name, container="coiled/notebook:latest", conda={ "channels": ["conda-forge"], "dependencies": ["python=3.8", "coiled=0.0.37", "dask=2021.3.0"] }, ) coiled.create_job_configuration( name="examples/quickstart", software=software_name, command=[ "/bin/bash", "run.sh", ], files=["quickstart.ipynb", "workspace.json", "run.sh"], ports=[8888], description="Quickly launch a Dask cluster on the cloud with Coiled", )