def get_environment( ws, environment_name, docker_image="todrabas/aml_rapids:latest", python_interpreter="/opt/conda/envs/rapids/bin/python", conda_packages=["matplotlib"], ): if environment_name not in ws.environments: env = Environment(name=environment_name) env.docker.enabled = True env.docker.base_image = docker_image env.python.interpreter_path = python_interpreter env.python.user_managed_dependencies = True conda_dep = CondaDependencies() for conda_package in conda_packages: conda_dep.add_conda_package(conda_package) env.python.conda_dependencies = conda_dep env.register(workspace=ws) else: env = ws.environments[environment_name] return env
def createOrGetEnvironment(ws, login_config, app_config): environment_name = login_config["aml_compute"]["environment_name"] python_interpreter = login_config["aml_compute"]["python_interpreter"] conda_packages = login_config["aml_compute"]["conda_packages"] ### CREATE OR RETRIEVE THE ENVIRONMENT if environment_name not in ws.environments: logger.info(f"Creating {environment_name} environment...") env = Environment(name=environment_name) env.docker.enabled = login_config["aml_compute"]["docker_enabled"] env.docker.base_image = None env.docker.base_dockerfile = f'FROM {app_config["base_dockerfile"]}' env.python.interpreter_path = python_interpreter env.python.user_managed_dependencies = True conda_dep = CondaDependencies() for conda_package in conda_packages: conda_dep.add_conda_package(conda_package) env.python.conda_dependencies = conda_dep env.register(workspace=ws) evn = env else: logger.info(f" Environment {environment_name} found...") env = ws.environments[environment_name] return env
def get_or_create_python_environment(azure_config: AzureConfig, source_config: SourceConfig, environment_name: str = "", register_environment: bool = True) -> Environment: """ Creates a description for the Python execution environment in AzureML, based on the Conda environment definition files that are specified in `source_config`. If such environment with this Conda environment already exists, it is retrieved, otherwise created afresh. :param azure_config: azure related configurations to use for model scale-out behaviour :param source_config: configurations for model execution, such as name and execution mode :param environment_name: If specified, try to retrieve the existing Python environment with this name. If that is not found, create one from the Conda files provided. This parameter is meant to be used when running inference for an existing model. :param register_environment: If True, the Python environment will be registered in the AzureML workspace. If False, it will only be created, but not registered. Use this for unit testing. """ # Merge the project-specific dependencies with the packages that InnerEye itself needs. This should not be # necessary if the innereye package is installed. It is necessary when working with an outer project and # InnerEye as a git submodule and submitting jobs from the local machine. # In case of version conflicts, the package version in the outer project is given priority. conda_dependencies, merged_yaml = merge_conda_dependencies(source_config.conda_dependencies_files) # type: ignore if azure_config.pip_extra_index_url: # When an extra-index-url is supplied, swap the order in which packages are searched for. # This is necessary if we need to consume packages from extra-index that clash with names of packages on # pypi conda_dependencies.set_pip_option(f"--index-url {azure_config.pip_extra_index_url}") conda_dependencies.set_pip_option("--extra-index-url https://pypi.org/simple") env_variables = { "AZUREML_OUTPUT_UPLOAD_TIMEOUT_SEC": str(source_config.upload_timeout_seconds), "MKL_SERVICE_FORCE_INTEL": "1", **(source_config.environment_variables or {}) } base_image = "mcr.microsoft.com/azureml/openmpi3.1.2-cuda10.2-cudnn8-ubuntu18.04" # Create a name for the environment that will likely uniquely identify it. AzureML does hashing on top of that, # and will re-use existing environments even if they don't have the same name. # Hashing should include everything that can reasonably change. Rely on hashlib here, because the built-in # hash function gives different results for the same string in different python instances. hash_string = "\n".join([merged_yaml, azure_config.docker_shm_size, base_image, str(env_variables)]) sha1 = hashlib.sha1(hash_string.encode("utf8")) overall_hash = sha1.hexdigest()[:32] unique_env_name = f"InnerEye-{overall_hash}" try: env_name_to_find = environment_name or unique_env_name env = Environment.get(azure_config.get_workspace(), name=env_name_to_find, version=ENVIRONMENT_VERSION) logging.info(f"Using existing Python environment '{env.name}'.") return env except Exception: logging.info(f"Python environment '{unique_env_name}' does not yet exist, creating and registering it.") env = Environment(name=unique_env_name) env.docker.enabled = True env.docker.shm_size = azure_config.docker_shm_size env.python.conda_dependencies = conda_dependencies env.docker.base_image = base_image env.environment_variables = env_variables if register_environment: env.register(azure_config.get_workspace()) return env
def _create_environment(ws, environment_name): env = Environment(workspace=ws, name=environment_name) env.docker.enabled = True env.docker.base_image = None env.docker.base_dockerfile = open( os.path.join(SRC_PATH, "Dockerfile.detector"), "r").read() env.python.user_managed_dependencies = True env.register(workspace=ws) return env
def create_or_update_environment(workspace, name, docker_image, docker_args=None): try: environment = Environment.get(workspace, name) cprint( 'Using existing environment "{}"'.format(colored(name, "white")), "green") except Exception: # MS - please implement an exception type for env not found cprint('Creating new environment "{}"'.format(colored(name, "white")), "yellow") environment = Environment(name=name) environment.docker.enabled = True environment.python.user_managed_dependencies = True # Handle dockerfile vs image spec accordingly if os.path.exists(docker_image): environment.docker.base_dockerfile = docker_image environment.docker.base_image = None else: environment.docker.base_dockerfile = None environment.docker.base_image = docker_image environment.docker.shm_size = "100g" if docker_args is not None: environment.docker.arguments = docker_args environment.python.user_managed_dependencies = True environment = environment.register(workspace=workspace) return environment
def create_conda_environment(workspace, name, conda_dependencies, pip_dependencies): """ Create an environment or retrieve it by its name from workspace Pip installs Python packages whereas conda installs packages which may contain software written in any language. e.g. TensorFlow, Scikit-Learn -> Conda, Matplotlib -> pip """ if name in Environment.list(workspace): env = Environment.get(workspace=workspace, name=name) print("The environment '{}' already existed for the workspace".format( name)) else: env = Environment(name=name) env.docker.enabled = True env.python.conda_dependencies = CondaDependencies.create( conda_packages=conda_dependencies, pip_packages=pip_dependencies, ) env.register(workspace=workspace) return env
def get_training_environment(ws: Workspace, name: str, pip_file: str, use_gpu: bool = False, include_prerelease: bool = False, environment_type: str = None): ''' Creates a training environment, based on the required pip packages, the need for GPU and a given environment type Args: ws (Workspace): the AzureML workspace that will be used to register the environment name (str): the name for the environment that will be registered use_gpu (bool): indicating if a GPU is required or not include_prerelease (bool): indicates if the pip packages can be installed in prerelease mode environment_type (str): either the name of an existing environment that will be taken as base, or one of these values (tensorflow, sklearn, pytorch). Returns: a registered environment , ready to use ''' from azureml.train.estimator import Estimator from azureml.core import Environment, ScriptRunConfig from azureml.core.runconfig import RunConfiguration from azureml.core.runconfig import CondaDependencies print('Getting environment for type', environment_type) base_environment = environment_type if (environment_type == 'tensorflow'): # Using Tensorflow Estimator base_environment = 'AzureML-TensorFlow-2.0-GPU' if use_gpu else 'AzureML-TensorFlow-2.0-CPU' elif (environment_type == 'sklearn'): base_environment = 'AzureML-Scikit-learn-0.20.3' elif (environment_type == 'pytorch'): base_environment = 'AzureML-PyTorch-1.5-GPU' if use_gpu else 'AzureML-PyTorch-1.5-GPU' pip_packages = __get_package_list_from_requirements(pip_file) if base_environment is not None: print('Taking', base_environment, 'as base environment') training_env = Environment.get(ws, base_environment) training_env.name = name for pippkg in pip_packages: training_env.python.conda_dependencies.add_pip_package(pippkg) else: print('Creating new environment') training_env = Environment(name=name) training_env.python.conda_dependencies = CondaDependencies.create( pip_packages=pip_packages) if (include_prerelease): training_env.python.conda_dependencies.set_pip_option("--pre") training_env.docker.enabled = True _ = training_env.register(workspace=ws) return training_env
# ----PYTHON ENV------ #------------------------- packages = CondaDependencies.create( conda_packages=["cudatoolkit=10.0"], pip_packages=[ 'azureml-sdk', 'PyYAML', 'azure-storage-blob', 'matplotlib', 'seaborn', 'tensorflow', 'Keras', 'tensorflow-hub', 'joblib', 'tqdm', 'Pillow', 'azureml-dataprep[pandas,fuse]>=1.1.14' ]) diagnoz_env = Environment("diagnoz-pipeline-env") diagnoz_env.python.user_managed_dependencies = False # Let Azure ML manage dependencies diagnoz_env.docker.enabled = True # Use a docker container diagnoz_env.docker.base_image = DEFAULT_GPU_IMAGE diagnoz_env.python.conda_dependencies = packages diagnoz_env.register(workspace=ws) # Runconfigs pipeline_run_config = RunConfiguration() pipeline_run_config.target = compute_target pipeline_run_config.environment = diagnoz_env print("Run configuration created.") shutil.rmtree(script_folder, ignore_errors=True) os.makedirs(script_folder, exist_ok=True) #copy all necessary scripts files = FilesProviders.get_path_files( "../", [os.path.basename(__file__), "__init__.py"]) for f in files:
conda_dep.add_pip_package("transformers") conda_dep.add_pip_package("matplotlib") conda_dep.add_pip_package("apex==0.9.10dev") conda_dep.add_pip_package("pandas") conda_dep.add_pip_package("pillow") conda_dep.add_pip_package("requests") conda_dep.add_pip_package("scikit-learn") conda_dep.add_pip_package("tqdm") bert_env.docker.enabled = True # bert_env.from_conda_specification('bert', './environment.yml') # bert_env.from_existing_conda_environment('bert_aml_env', 'bert') bert_env.python.conda_dependencies = conda_dep # bert_env.python.conda_dependencies = conda_dep bert_env.register(ws) # Choose a name for your CPU cluster gpu_cluster_name = "gpu-compute-bert" # Verify that cluster does not exist already try: gpu_cluster = ComputeTarget(workspace=ws, name=gpu_cluster_name) print('Found existing cluster, use it.') except ComputeTargetException: compute_config = AmlCompute.provisioning_configuration( vm_size='STANDARD_NC6', max_nodes=4) gpu_cluster = ComputeTarget.create(ws, gpu_cluster_name, compute_config) gpu_cluster.wait_for_completion(show_output=True)
pipeline_cluster.wait_for_completion(show_output=True) # Create a Python environment for the experiment fraud_env = Environment("fraud-pipeline-env") fraud_env.python.user_managed_dependencies = False # Let Azure ML manage dependencies fraud_env.docker.enabled = True # Use a docker container # Create a set of package dependencies fraud_packages = CondaDependencies.create( conda_packages=['scikit-learn', 'pandas'], pip_packages=['azureml-sdk']) # Add the dependencies to the environment fraud_env.python.conda_dependencies = fraud_packages # Register the environment (just in case you want to use it again) fraud_env.register(workspace=ws) registered_env = Environment.get(ws, 'fraud-pipeline-env') # Create a new runconfig object for the pipeline pipeline_run_config = RunConfiguration() # Use the compute you created above. pipeline_run_config.target = pipeline_cluster # Assign the environment to the run configuration pipeline_run_config.environment = registered_env print("Run configuration created.") # Get the training dataset fraud_ds = ws.datasets.get("creditcard")
def main(): parser = argparse.ArgumentParser( description="NGC Set Up on AzureML Compute Cluster") parser.add_argument("--config_file", type=str, help="location of config file") args = parser.parse_args() config_file = args.config_file print(config_file) configdata = ngccontent.get_config(config_file) subscription_id = configdata["azureml_user"]["subscription_id"] resource_group = configdata["azureml_user"]["resource_group"] workspace_name = configdata["azureml_user"]["workspace_name"] ws = Workspace(workspace_name=workspace_name, subscription_id=subscription_id, resource_group=resource_group) verify = f''' Subscription ID: {subscription_id} Resource Group: {resource_group} Workspace: {workspace_name}''' print(verify) ### vnet settings vnet_rg = ws.resource_group vnet_name = configdata["aml_compute"]["vnet_name"] subnet_name = configdata["aml_compute"]["subnet_name"] ### azure ml names ct_name = configdata["aml_compute"]["ct_name"] exp_name = configdata["aml_compute"]["exp_name"] ### trust but verify verify = f''' vNET RG: {vnet_rg} vNET name: {vnet_name} vNET subnet name: {subnet_name} Compute target: {ct_name} Experiment name: {exp_name}''' print(verify) if configdata["aml_compute"]["vm_name"] in configdata[ "supported_vm_sizes"].keys(): vm_name = configdata["aml_compute"]["vm_name"] gpus_per_node = configdata["supported_vm_sizes"][vm_name] print( "Setting up compute target {ct_name} with vm_size: {vm_name} with {gpus_per_node} GPUs" .format(ct_name=ct_name, vm_name=vm_name, gpus_per_node=gpus_per_node)) if ct_name not in ws.compute_targets: config = AmlCompute.provisioning_configuration( vm_size=vm_name, min_nodes=configdata["aml_compute"]["min_nodes"], max_nodes=configdata["aml_compute"]["max_nodes"], vnet_resourcegroup_name=vnet_rg, vnet_name=vnet_name, subnet_name=subnet_name, idle_seconds_before_scaledown=configdata["aml_compute"] ["idle_seconds_before_scaledown"], remote_login_port_public_access='Enabled') ct = ComputeTarget.create(ws, ct_name, config) ct.wait_for_completion(show_output=True) else: print("Loading Pre-existing Compute Target {ct_name}".format( ct_name=ct_name)) ct = ws.compute_targets[ct_name] else: print("Unsupported vm_size {vm_size}".format(vm_size=vm_name)) print("The specified vm size must be one of ...") for azure_gpu_vm_size in configdata["supported_vm_sizes"].keys(): print("... " + azure_gpu_vm_size) raise Exception( "{vm_size} does not support Pascal or above GPUs".format( vm_size=vm_name)) environment_name = configdata["aml_compute"]["environment_name"] python_interpreter = configdata["aml_compute"]["python_interpreter"] conda_packages = configdata["aml_compute"]["conda_packages"] from azureml.core import ContainerRegistry if environment_name not in ws.environments: env = Environment(name=environment_name) env.docker.enabled = configdata["aml_compute"]["docker_enabled"] env.docker.base_image = None env.docker.base_dockerfile = "FROM {dockerfile}".format( dockerfile=configdata["ngc_content"]["base_dockerfile"]) env.python.interpreter_path = python_interpreter env.python.user_managed_dependencies = True conda_dep = CondaDependencies() for conda_package in conda_packages: conda_dep.add_conda_package(conda_package) env.python.conda_dependencies = conda_dep env.register(workspace=ws) evn = env else: env = ws.environments[environment_name] amlcluster = AzureMLComputeCluster( workspace=ws, compute_target=ct, initial_node_count=1, experiment_name=configdata["aml_compute"]["exp_name"], environment_definition=env, use_gpu=True, n_gpus_per_node=1, jupyter=True, jupyter_port=configdata["aml_compute"]["jupyter_port"], dashboard_port=9001, scheduler_port=9002, scheduler_idle_timeout=1200, worker_death_timeout=30, additional_ports=[], datastores=[], telemetry_opt_out=True, asynchronous=False) print(amlcluster.jupyter_link) amlcluster.jupyter_link print('Exiting script')
pip_packages=['azureml-defaults'] env.python.conda_dependencies = deps #override w custom img env.docker.base_image='my-base-image' env.docker.base_image_registry='myregistry.azurecr.io/myimage' #override how AZ auto handles pkg dependencies etc env.python.user_managed_dependencies=True env.python.interpreter_path = '/opt/miniconda/bin/python' #register env.register(workspace=ws) #view registered from azureml.core import Environment env_names = Environment.list(workspace=ws) for env_name in env_names: print('Name:',env_name) #retrieve enviro from azureml.core import Environment from azureml.train.estimator import Estimator training_env = Environment.get(workspace=ws, name='training_environment') estimator = Estimator(source_directory='experiment_folder' entry_script='training_script.py',
def get_environment( workspace: Workspace, environment_name: str, conda_dependencies_file: str = None, create_new: bool = False, enable_docker: bool = None, docker_image: str = None, dockerfile: str = None, use_gpu: bool = False, ): try: e = Env() environments = Environment.list(workspace=workspace) restored_environment = None for env in environments: if env == environment_name: restored_environment = environments[environment_name] if restored_environment is None or create_new: # Environment has to be created if conda_dependencies_file is not None: new_env = Environment.from_conda_specification( environment_name, os.path.join(e.sources_directory_train, conda_dependencies_file), # NOQA: E501 ) # NOQA: E501 restored_environment = new_env else: restored_environment = Environment(environment_name) if enable_docker is not None: restored_environment.docker.enabled = enable_docker if docker_image is not None: restored_environment.docker.base_image = docker_image # In case of own image # don't append AML managed dependencies restored_environment.python.\ user_managed_dependencies = True elif dockerfile is not None: # Alternatively, load from a file. with open(dockerfile, "r") as f: dockerfile = f.read() restored_environment.docker.\ base_dockerfile = dockerfile # In case of own Dockerfile # don't append AML managed dependencies restored_environment.python.\ user_managed_dependencies = True else: restored_environment.docker.\ base_image = DEFAULT_GPU_IMAGE if use_gpu else DEFAULT_CPU_IMAGE # NOQA: E501 restored_environment.register(workspace) if restored_environment is not None: print(restored_environment) return restored_environment except Exception as e: print(e) exit(1)
# Create a Python environment for the experiment model_env = Environment(env_name) model_env.python.user_managed_dependencies = False # Let Azure ML manage dependencies model_env.docker.enabled = True # Use a docker container # Create a set of package dependencies packages = CondaDependencies.create(conda_packages=['scikit-learn','pandas'], pip_packages=['azureml-sdk']) # Add the dependencies to the environment model_env.python.conda_dependencies = packages # Register the environment (just in case you want to use it again) model_env.register(workspace=ws) registered_env = Environment.get(ws, env_name) # Create a new runconfig object for the pipeline pipeline_run_config = RunConfiguration() # Use the compute you created above. pipeline_run_config.target = pipeline_cluster # Assign the environment to the run configuration pipeline_run_config.environment = registered_env print ("Run configuration created.")
"requests==2.21.0", "sklearn", "pandas", "numpy", "pillow==6.0.0", "tensorflow-gpu==1.15", "keras", "matplotlib", "seaborn", ]) env = Environment("prednet") env.python.conda_dependencies = conda_dependencies env.docker.enabled = True env.docker.base_image = DEFAULT_GPU_IMAGE env.register(ws) # Runconfigs runconfig = RunConfiguration() runconfig.environment = env print("PipelineData object created") create_pipelines = PythonScriptStep( name="create pipelines", script_name="pipelines_slave.py", compute_target=cpu_compute_target, arguments=[ "--cpu_compute_name", cpu_compute_name, "--gpu_compute_name", gpu_compute_name ], source_directory=script_folder,
freezer_environment = Environment("sktime_freezer_environment") cd = CondaDependencies.create( conda_packages=["numpy", "cython", "pandas", "scikit-learn"], pip_packages=[ "azureml-defaults", "inference-schema[numpy-support]", "joblib==0.13.*", "azureml-dataprep[pandas, fuse]", "sktime", ], ) freezer_environment.docker.enabled = True freezer_environment.docker.base_image = DEFAULT_CPU_IMAGE freezer_environment.python.conda_dependencies = cd freezer_environment.register(workspace=ws) logger.info("Environment registered") try: cpu_cluster = ComputeTarget(workspace=ws, name="freezertrain") logger.info("Found existing compute target") except ComputeTargetException: logger.info("Creating a new compute target...") cpu_cluster = ComputeTarget.create( ws, "freezertrain", AmlCompute.provisioning_configuration(vm_size="STANDARD_DS3_V2", max_nodes=8), ) cpu_cluster.wait_for_completion( show_output=True, min_node_count=None, timeout_in_minutes=20
def get_environment(workspace, name, pip_requirements=None, conda_specification=None, conda_env=None, docker_image=None, docker_file=None, override=False, inference_stack=None): """ Get an Azure ML environment from PIP or Conda. From: - pip_requirements - conda_specification - conda_env at most one can be provided. If none is provided, it is assumed that the requirements are taken care of by the user. From: - docker_image - docker_file at most one can be provided. If none is provided, the base Azure image is used. :params workspace: The Azure ML workspace to look for existing environments. :params name: Name for this environment :params pip_requirements: Path to the pip requirements file :params conda_specifidation: Path to the conda specification file :params conda_env: Name of the conda environment to use :params docker_image: Base the image off an existing docker image :params docker_file: Base the image off a Dockerfile. :params override: Create a new environment with this name, regardless of if one already exists. :params inference_stack: Add a stack that enables this environment for inference. "latest" is a valid option. Set to None to not add this. :returns: Azure ML environment or None in case of failure """ if not override: try: env = Environment.get(workspace, name) print("Existing environment found, using that") return env except: print("No environment with that name found, creating new one") # Validate at most one of pip_requirements, conda_specification, conda_env # is provided if sum([ 1 for x in [pip_requirements, conda_specification, conda_env] if x is not None ]) > 1: print("Provide at most 1 of pip_requirements, conda_specification, " "conda_env") return None # Validate that at most one of docker_image, docker_file is # provided if sum([1 for x in [docker_image, docker_file] if x is not None]) > 1: print("Provide at most 1 of docker_image, docker_file") return None if pip_requirements is not None: env = Environment.from_pip_requirements(name, pip_requirements) elif conda_specification is not None: env = Environment.from_conda_specification(name, conda_specification) elif conda_env is not None: env = Environment.from_existing_conda_environment(name, conda_env) else: env = Environment(name) env.python.user_managed_dependencies = True if docker_file is not None: env.docker.enabled = True env.docker.base_image = None env.docker.base_dockerfile = docker_file elif docker_image is not None: env.docker.enabled = True env.docker.base_image = docker_image if inference_stack is not None: env.inferencing_stack_version = inference_stack # Register environment env.register(workspace=workspace) return env
from azureml.core import Workspace ws = Workspace.from_config() from azureml.core import Environment from azureml.core.conda_dependencies import CondaDependencies #created an env my_env = Environment("My_new_env") conda_dep = CondaDependencies.create(conda_packages=['scikit-learn']) my_env.python.conda_dependencies = conda_dep my_env.register() #creating the cluster from azureml.core.compute import AmlCompute cluster_config = AmlCompute.provisioning_configuration(vm_size="STANDARD_D11_V2"), max_nodes = 2) cluster = AmlCompute.create(ws, "My_cluster", cluster_config) cluster.wait_for_completion() #fetching the data input_ds = ws.datasets.get("Loan Application") #for ScriptRunning from azureml.core import ScriptRunConfig, Experiment script_run = ScriptRunConfig(source_directory = ".",
def main(): e = Env() print('********************') print(e.source_directory) files = os.listdir('./aml_pipeline') for f in files: print(f) print('***************') workspace_name = e.workspace_name subscription_id = e.subscription_id resource_group = e.resource_group #Connect to AML Workspace print('workspace_name = ' + workspace_name) print('subscription_id = ' + subscription_id) print('resource_group = ' + resource_group) ws = Workspace.get( name=workspace_name, subscription_id=subscription_id, resource_group=resource_group, ) print('Ready to use Azure ML {} to work with {}'.format( azureml.core.VERSION, ws.name)) default_ds = ws.get_default_datastore() if 'diabetes dataset' not in ws.datasets: default_ds.upload_files( files=['diabetes.csv', 'diabetes2.csv'], # Upload the diabetes csv files in /data target_path= 'diabetes-data/', # Put it in a folder path in the datastore overwrite=True, # Replace existing files of the same name show_progress=True) #Create a tabular dataset from the path on the datastore (this may take a short while) tab_data_set = Dataset.Tabular.from_delimited_files( path=(default_ds, 'diabetes-data/*.csv')) # Register the tabular dataset try: tab_data_set = tab_data_set.register(workspace=ws, name='diabetes dataset', description='diabetes data', tags={'format': 'CSV'}, create_new_version=True) print('Dataset registered.') except Exception as ex: print(ex) else: print('Dataset already registered.') # Create a folder for the pipeline step files experiment_folder = 'diabetes_pipeline' os.makedirs(experiment_folder, exist_ok=True) print(experiment_folder) cluster_name = "mmcomputecluster" try: # Check for existing compute target pipeline_cluster = ComputeTarget(workspace=ws, name=cluster_name) print('Found existing cluster, use it.') except ComputeTargetException: # If it doesn't already exist, create it try: compute_config = AmlCompute.provisioning_configuration( vm_size='STANDARD_DS11_V2', max_nodes=2) pipeline_cluster = ComputeTarget.create(ws, cluster_name, compute_config) pipeline_cluster.wait_for_completion(show_output=True) except Exception as ex: print(ex) # Create a Python environment for the experiment diabetes_env = Environment("diabetes-pipeline-env") diabetes_env.python.user_managed_dependencies = False # Let Azure ML manage dependencies diabetes_env.docker.enabled = True # Use a docker container # Create a set of package dependencies diabetes_packages = CondaDependencies.create( conda_packages=[ 'scikit-learn', 'ipykernel', 'matplotlib', 'pandas', 'pip' ], pip_packages=[ 'azureml-defaults', 'azureml-dataprep[pandas]', 'pyarrow' ]) # Add the dependencies to the environment diabetes_env.python.conda_dependencies = diabetes_packages # Register the environment diabetes_env.register(workspace=ws) registered_env = Environment.get(ws, 'diabetes-pipeline-env') # Create a new runconfig object for the pipeline pipeline_run_config = RunConfiguration() # Use the compute you created above. pipeline_run_config.target = pipeline_cluster # Assign the environment to the run configuration pipeline_run_config.environment = registered_env print("Run configuration created.") # Get the training dataset diabetes_ds = ws.datasets.get("diabetes dataset") # Create a PipelineData (temporary Data Reference) for the model folder prepped_data_folder = PipelineData("prepped_data_folder", datastore=ws.get_default_datastore()) # Step 1, Run the data prep script prep_step = PythonScriptStep(name="Prepare Data", script_name="prep_diabetes.py", source_directory='./aml_pipeline', arguments=[ '--input-data', diabetes_ds.as_named_input('raw_data'), '--prepped-data', prepped_data_folder ], outputs=[prepped_data_folder], compute_target=pipeline_cluster, runconfig=pipeline_run_config, allow_reuse=True) # Step 2, run the training script train_step = PythonScriptStep( name="Train and Register Model", source_directory='./aml_pipeline', script_name="train_diabetes.py", arguments=['--training-folder', prepped_data_folder], inputs=[prepped_data_folder], compute_target=pipeline_cluster, runconfig=pipeline_run_config, allow_reuse=True) print("Pipeline steps defined") pipeline_steps = [prep_step, train_step] pipeline = Pipeline(workspace=ws, steps=pipeline_steps) print("Pipeline is built.") # Create an experiment and run the pipeline experiment = Experiment(workspace=ws, name='jlg-exp') pipeline_run = experiment.submit(pipeline, regenerate_outputs=True) print("Pipeline submitted for execution.") pipeline_run.wait_for_completion(show_output=True) for run in pipeline_run.get_children(): print(run.name, ':') metrics = run.get_metrics() for metric_name in metrics: print('\t', metric_name, ":", metrics[metric_name]) for model in Model.list(ws): print(model.name, 'version:', model.version) for tag_name in model.tags: tag = model.tags[tag_name] print('\t', tag_name, ':', tag) for prop_name in model.properties: prop = model.properties[prop_name] print('\t', prop_name, ':', prop) print('\n') # Publish the pipeline from the run published_pipeline = pipeline_run.publish_pipeline( name="diabetes-training-pipeline", description="Trains diabetes model", version="1.0") published_pipeline rest_endpoint = published_pipeline.endpoint print(rest_endpoint)
env = Environment("component-condition") env.docker.enabled = True cd = CondaDependencies.create( conda_packages=[ "tensorflow=2.0.0", "pandas", "numpy", "matplotlib" ], pip_packages=[ "azureml-mlflow==1.5.0", "azureml-defaults==1.5.0" ] ) env.python.conda_dependencies = cd env.register(workspace=ws) print("Registered environment component-condition") # Specify the run configuration run_config = RunConfiguration() run_config.environment.docker.enabled = True run_config.environment.python.conda_dependencies = cd # Pipeline definition inputdata = DataReference( datastore=Datastore.get(ws, "trainingdata"), data_reference_name="data" ) train_model = PythonScriptStep( script_name="./train.py",
from azureml.core import Workspace, Experiment, ScriptRunConfig ws = Workspace.from_config() experiment = Experiment(ws, "training-exp-02") from azureml.core import Environment from azureml.core.environment import CondaDependencies #creating Custom Environment my_env = Environment(name="MyEnvironment2") #creating the Dependencies my_env_dep = CondaDependencies.create(conda_packages=['scikit-learn']) my_env.python.conda_dependencies = my_env_dep #registering the Environment my_env.register(ws) script_config = ScriptRunConfig(source_directory=".", script="02-training_script.py", environment=my_env) new_run = experiment.submit(config=script_config) new_run.wait_for_completion()
def main(): e = Env() from azureml.core.authentication import InteractiveLoginAuthentication myten=os.environ.get("AZURE_TENANT_ID") interactive_auth = InteractiveLoginAuthentication(tenant_id=os.environ.get("AZURE_TENANT_ID")) subscription=os.environ.get("CSUBSCRIPTION") workspace_name=e.workspace_name resource_group=e.resource_group aml_workspace = Workspace.get( name = workspace_name, subscription_id = subscription, resource_group=resource_group, auth=interactive_auth ) from ml_service.util.attach_compute import get_compute # Get Azure machine learning cluster # If not present then get_compute will create a compute based on environment variables aml_compute = get_compute( aml_workspace, e.compute_name, e.vm_size) if aml_compute is not None: print("aml_compute:") print(aml_compute) print("SDK version: ", azureml.core.VERSION) ## Variable names that can be passed in as parameter values from azureml.pipeline.core.graph import PipelineParameter from azureml.core import Datastore model_name_param = PipelineParameter( name="model_name", default_value=e.model_name) dataset_version_param = PipelineParameter( name="dataset_version", default_value=e.dataset_version) data_file_path_param = PipelineParameter( name="data_file_path", default_value="none") caller_run_id_param = PipelineParameter( name="caller_run_id", default_value="none") #model_path = PipelineParameter( # name="model_path", default_value=e.model_path) if (e.datastore_name): datastore_name = e.datastore_name else: datastore_name = aml_workspace.get_default_datastore().name # Get the datastore whether it is the default or named store datastore = Datastore.get(aml_workspace, datastore_name) dataset_name = e.dataset_name # Create a reusable Azure ML environment from ml_service.util.manage_environment import get_environment from azureml.core import Environment # RUN Configuration ## Must have this process to work with AzureML-SDK 1.0.85 from azureml.core.runconfig import RunConfiguration, DEFAULT_CPU_IMAGE from azureml.core.conda_dependencies import CondaDependencies try: app_env=Environment(name="smartschedule_env") app_env.register(workspace=aml_workspace) except: print("Environment not found") # Create a new runconfig object aml_run_config = RunConfiguration() aml_run_config.environment.environment_variables["DATASTORE_NAME"] = e.datastore_name # NOQA: E501 # Use the aml_compute you created above. aml_run_config.target = aml_compute # Enable Docker aml_run_config.environment.docker.enabled = True # Set Docker base image to the default CPU-based image aml_run_config.environment.docker.base_image = DEFAULT_CPU_IMAGE #aml_run_config.environment.docker.base_image = "mcr.microsoft.com/azureml/base:0.2.1" # Use conda_dependencies.yml to create a conda environment in the Docker image for execution aml_run_config.environment.python.user_managed_dependencies = False app_conda_deps=CondaDependencies.create( conda_packages=['pandas','scikit-learn', 'libgcc','pyodbc', 'sqlalchemy', 'py-xgboost==0.90'], pip_packages=['azureml-sdk[automl,explain,contrib,interpret]==1.4.0', 'xgboost==0.90', 'azureml-dataprep==1.4.6', 'pyarrow', 'azureml-defaults==1.4.0', 'azureml-train-automl-runtime==1.4.0'], pin_sdk_version=False) # Specify CondaDependencies obj, add necessary packages aml_run_config.environment.python.conda_dependencies = app_conda_deps print ("Run configuration created.") from azure.common.credentials import ServicePrincipalCredentials #from azure.keyvault import KeyVaultClient, KeyVaultAuthentication from azure.keyvault.secrets import SecretClient from azure.identity import DefaultAzureCredential import pandas as pd #import sqlalchemy as sql import pyodbc def get_data(sql_string, columns): credentials = None credential = DefaultAzureCredential() secret_client = SecretClient("https://smrtschd-aml-kv.vault.azure.net", credential=credential) secret = secret_client.get_secret("database-connection") #client = KeyVaultClient(KeyVaultAuthentication(auth_callback)) #secret_bundle = client.get_secret("https://smrtschd-aml-kv.vault.azure.net", "database-connection", "") server = 'starlims-sql.database.windows.net' database = 'QM12_DATA_AUTOMATION' username = '******' password = secret.value driver= '{ODBC Driver 17 for SQL Server}' conn = pyodbc.connect('Driver='+driver+';'+ 'Server='+server+';'+ 'Database='+database+';'+ 'PORT=1433;'+ 'UID='+username+';'+ 'PWD='+password+'; MARS_Connection=Yes' ) try: SQL_Query = pd.read_sql_query(sql_string, conn) df = pd.DataFrame(SQL_Query, columns=columns) return df except Exception as e: print(e) raise sql_str = "SELECT " \ " Dept " \ ", Method " \ ", Servgrp " \ ", Runno " \ ", TestNo " \ ", Testcode " \ ", Total_Duration_Min " \ ", Total_Duration_Hr " \ ", Usrnam " \ ", Eqid " \ ", Eqtype " \ "FROM dbo.Draft " \ "order by TESTCODE, RUNNO, dept, method;" columns = ["Dept", "Method", "Servgrp", "Runno", "TestNo", "Testcode", "Total_Duration_Min", "Total_Duration_Hr", "Usrnam", "Eqid","Eqtype"] from azureml.core import Dataset from sklearn.model_selection import train_test_split if (e.train_dataset_name not in aml_workspace.datasets): df = get_data(sql_str, columns) train_df, test_df=train_test_split(df, test_size=0.2) MY_DIR = "data" CHECK_FOLDER = os.path.isdir(MY_DIR) if not CHECK_FOLDER: os.makedirs(MY_DIR) else: print("Folder ", MY_DIR, " is already created") #files = ["data/analyst_tests.csv"] files = ["data/train_data.csv","data/test_data.csv"] def_file_store = Datastore(aml_workspace, "workspacefilestore") dtfrm = df.to_csv(files[0], header=True, index=False) train_dataframe=train_df.to_csv(files[0], header=True, index=False) test_dataframe=test_df.to_csv(files[1], header=True, index=False) datastore.upload_files( files=files, target_path='data/', overwrite=True ) from azureml.data.data_reference import DataReference blob_input_data_test=DataReference( datastore=datastore, data_reference_name="smartschedulertest", path_on_datastore="data/test_data.csv" ) test_data=Dataset.Tabular.from_delimited_files(blob_input_data_test) test_data.register(aml_workspace, e.test_dataset_name, create_new_version=True) blob_input_data_train=DataReference( datastore=datastore, data_reference_name="smartschedulertrain", path_on_datastore="data/train_data.csv" ) train_data=Dataset.Tabular.from_delimited_files(blob_input_data_train) train_data.register(aml_workspace, e.train_dataset_name, create_new_version=True) else: from azureml.data.data_reference import DataReference print("getting from the datastore instead of uploading") train_data=Dataset.get_by_name(aml_workspace, name=e.train_dataset_name) test_data=Dataset.get_by_name(aml_workspace, name=e.test_dataset_name) # check the training dataset to make sure it has at least 50 records. tdf=train_data.to_pandas_dataframe().head(5) print(tdf.shape) print(tdf) # display the first five rows of the data # create a variable that can be used for other purposes df=train_data.to_pandas_dataframe().head() label_column="Total_Duration_Min" import random import string def randomString(stringLength=15): letters = string.ascii_lowercase return ''.join(random.choice(letters) for i in range(stringLength)) from azureml.core import Experiment experiment = Experiment(aml_workspace, "SmartScheduler_Pipeline") import logging aml_name = 'smart_scheduler_' + randomString(5) print(aml_name) import numpy as np import matplotlib.pyplot as plt from matplotlib.ticker import StrMethodFormatter print(df.head(5)) print(df.shape) print(df.dtypes) #df.hist(column='Dept') list(df.columns.values) # Remove Features that are not necessary. #df.hist(column="Servgrp", bins=4) train_data=train_data.drop_columns(["Runno","TestNo","Total_Duration_Hr"]) test_data=test_data.drop_columns(["Runno","TestNo","Total_Duration_Hr"]) print(train_data.to_pandas_dataframe()) print(test_data.to_pandas_dataframe()) from azureml.automl.core.featurization import FeaturizationConfig # some of the columns could be change to one hot encoding especially if the categorical column featurization_config=FeaturizationConfig() featurization_config.blocked_transformers=['LabelEncoder'] featurization_config.add_column_purpose('Dept', 'CategoricalHash') featurization_config.add_transformer_params('HashOneHotEncoder',['Method'], {"number_of_bits":3}) featurization_config.add_column_purpose('Servgrp', 'CategoricalHash') featurization_config.add_column_purpose('Testcode', 'Numeric') featurization_config.add_column_purpose('Usrnam', 'CategoricalHash') featurization_config.add_column_purpose('Eqid', 'CategoricalHash') featurization_config.add_column_purpose('Eqtype', 'CategoricalHash') from azureml.pipeline.core import Pipeline, PipelineData from azureml.pipeline.steps import PythonScriptStep #train_model_folder = './scripts/trainmodel' automl_settings = { "iteration_timeout_minutes": 5, "iterations": 5, "enable_early_stopping": True, "primary_metric": 'spearman_correlation', "verbosity": logging.INFO, "n_cross_validation":5 } automl_config = AutoMLConfig(task="regression", debug_log='automated_ml_errors.log', #path = train_model_folder, training_data=train_data, featurization=featurization_config, blacklist_models=['XGBoostRegressor'], label_column_name=label_column, compute_target=aml_compute, **automl_settings) from azureml.pipeline.steps import AutoMLStep from azureml.pipeline.core import TrainingOutput metrics_output_name = 'metrics_output' best_model_output_name='best_model_output' metrics_data = PipelineData(name = 'metrics_data', datastore = datastore, pipeline_output_name=metrics_output_name, training_output=TrainingOutput(type='Metrics')) model_data = PipelineData(name='model_data', datastore=datastore, pipeline_output_name=best_model_output_name, training_output=TrainingOutput(type='Model')) trainWithAutomlStep = AutoMLStep( name=aml_name, automl_config=automl_config, passthru_automl_config=False, outputs=[metrics_data, model_data], allow_reuse=True ) evaluate_step = PythonScriptStep( name="Evaluate Model", script_name='./evaluate/evaluate_model.py', # e.evaluate_script_path, compute_target=aml_compute, source_directory='../app', arguments=[ "--model_name", model_name_param, "--allow_run_cancel", e.allow_run_cancel ] ) register_step = PythonScriptStep( name="Register Model ", script_name='register/register_model2.py', #e.register_script_path, compute_target=aml_compute, source_directory='../app', inputs=[model_data], arguments=[ "--model_name", model_name_param, "--model_path", model_data, "--ds_name", e.train_dataset_name ], runconfig=aml_run_config, allow_reuse=False ) if ((e.run_evaluation).lower() == 'true'): print("Include evaluation step before register step.") evaluate_step.run_after(trainWithAutomlStep) register_step.run_after(evaluate_step) pipeline_steps = [ trainWithAutomlStep, evaluate_step, register_step ] else: print("Exclude the evaluation step and run register step") register_step.run_after(trainWithAutomlStep) pipeline_steps = [ trainWithAutomlStep, register_step ] print( "this is the value for execute pipeline: {}".format(e.execute_pipeline)) if( (e.execute_pipeline).lower() =='true' ): # Execute the pipe normally during testing and debugging print("Pipeline submitted for execution.") pipeline = Pipeline(workspace = aml_workspace, steps=pipeline_steps) pipeline_run = experiment.submit(pipeline) pipeline_run.wait_for_completion() print("Pipeline is built.") else: # Generates pipeline that will be called in ML Ops train_pipeline = Pipeline(workspace=aml_workspace, steps=pipeline_steps) train_pipeline._set_experiment_name train_pipeline.validate() published_pipeline = train_pipeline.publish( name=e.pipeline_name, description="Model training/retraining pipeline", version=e.build_id ) print(f'Published pipeline: {published_pipeline.name}') print(f'for build {published_pipeline.version}')