# set Docker base image to the default CPU-based image run_config.environment.docker.base_image = azureml.core.runconfig.DEFAULT_MMLSPARK_CPU_IMAGE print('base image is', run_config.environment.docker.base_image) #run_config.environment.docker.base_image = 'microsoft/mmlspark:plus-0.9.9' # use conda_dependencies.yml to create a conda environment in the Docker image for execution # please update this file if you need additional packages. run_config.environment.python.user_managed_dependencies = False cd = CondaDependencies() cd.add_conda_package('numpy') # overwrite the default conda_dependencies.yml file cd.save_to_file(project_dir = project_folder, file_name='conda_dependencies.yml') # auto-prepare the Docker image when used for execution (if it is not already prepared) run_config.prepare_environment = True print() print('##################################################') print('submitting {} for a Spark run on ACI...'.format(train_script)) print('##################################################') print() run = Run.submit(project_object = project, run_config = run_config, script_to_run = "train-spark.py") print(helpers.get_run_history_url(run)) # Shows output of the run on stdout. run.wait_for_completion(show_output = True)
vm_size="STANDARD_D2_V2", # for GPU, use "STANDARD_NC6" #vm_priority = 'lowpriority', # optional autoscale_enabled=True, cluster_min_nodes=1, cluster_max_nodes=4) # create the cluster compute_target = ws.create_compute_target(batchai_cluster_name, provisioning_config) compute_target.wait_for_provisioning(show_output=True) print('create Batch AI run config') rc = RunConfiguration(project, "dask_run_config") rc.environment.docker.enabled = True rc.prepare_environment = True rc.target = batchai_cluster_name rc.environment.python.user_managed_dependencies = False rc.batchai.node_count = 2 # create a new CondaDependencies obj cd = CondaDependencies() # add scikit-learn as a conda dependency cd.add_conda_package('dask') cd.add_conda_package('joblib') cd.add_pip_package('azureml-contrib-daskonbatch') # overwrite the default conda_dependencies.yml file cd.save_to_file(project_dir=project_folder, file_name='conda_dependencies.yml') print()
cli_auth = AzureCliAuthentication() # Get workspace ws = Workspace.from_config(auth=cli_auth) # Attach Experiment experiment_name = "devops-ai-demo" exp = Experiment(workspace=ws, name=experiment_name) print(exp.name, exp.workspace.name, sep="\n") # Editing a run configuration property on-fly. run_config_system_managed = RunConfiguration() # Use a new conda environment that is to be created from the conda_dependencies.yml file run_config_system_managed.environment.python.user_managed_dependencies = False # Automatically create the conda environment before the run run_config_system_managed.prepare_environment = True # # add scikit-learn to the conda_dependencies.yml file # Specify conda dependencies with scikit-learn # run_config_system_managed.environment.python.conda_dependencies = CondaDependencies.create(conda_packages=['scikit-learn']) print("Submitting an experiment to new conda virtual env") src = ScriptRunConfig( source_directory="./code", script="training/train.py", run_config=run_config_user_managed, ) run = exp.submit(src) # Shows output of the run on stdout. run.wait_for_completion(show_output=True, wait_post_processing=True)
run_dsvm = RunConfiguration(framework = "python") # Set the compute target to the Linux DSVM run_dsvm.target = compute_target_name # Use Docker in the remote VM run_dsvm.environment.docker.enabled = True # Use the CPU base image # To use GPU in DSVM, you must also use the GPU base Docker image "azureml.core.runconfig.DEFAULT_GPU_IMAGE" run_dsvm.environment.docker.base_image = azureml.core.runconfig.DEFAULT_CPU_IMAGE print('Base Docker image is:', run_dsvm.environment.docker.base_image) # Prepare the Docker and conda environment automatically when they're used for the first time run_dsvm.prepare_environment = True # Specify the CondaDependencies object run_dsvm.environment.python.conda_dependencies = CondaDependencies.create(conda_packages=['scikit-learn']) #</run_dsvm> hdi_compute.name = "blah" from azureml.core.runconfig import RunConfiguration from azureml.core.conda_dependencies import CondaDependencies # use pyspark framework hdi_run_config = RunConfiguration(framework="pyspark") # Set compute target to the HDI cluster hdi_run_config.target = hdi_compute.name