def create_env_from_requirements(file_path='requirements.txt', name='mlapp', endpoint=False): env = Environment.from_pip_requirements(name=name, file_path=file_path) # Enable Docker env.docker.enabled = True # Set Docker base image to the default CPU-based image path_to_dockerfile = os.path.join(os.getcwd(), 'deployment', 'Dockerfile') if os.path.exists(path_to_dockerfile): # Set custom Docker image env.docker.base_image = None env.docker.base_dockerfile = path_to_dockerfile else: # Set Docker base image to the default CPU-based image env.docker.base_image = DEFAULT_CPU_IMAGE # Use conda_dependencies.yml to create a conda environment in the Docker image for execution env.python.user_managed_dependencies = False if endpoint: env.inferencing_stack_version = 'latest' return env
def transformers_environment(use_gpu=True): """Prepares Azure ML Environment with transformers library. Note: We install transformers library from source. See requirements file for full list of dependencies. Args: use_gpu (bool): If true, Azure ML will use gpu-enabled docker image as base. Return: Azure ML Environment with huggingface libraries needed to perform GLUE finetuning task. """ pip_requirements_path = str( Path(__file__).parent.joinpath("requirements.txt")) print(f"Create Azure ML Environment from {pip_requirements_path}") if use_gpu: env_name = "transformers-gpu" env = Environment.from_pip_requirements( name=env_name, file_path=pip_requirements_path, ) env.docker.base_image = ( "mcr.microsoft.com/azureml/intelmpi2018.3-cuda10.0-cudnn7-ubuntu16.04" ) else: env_name = "transformers-cpu" env = Environment.from_pip_requirements( name=env_name, file_path=pip_requirements_path, ) return env
def get_environment(self, config): name = config.get("name") pip_wheel_path = config.get("pip_wheel_path") requirements_path = config.get("requirements_path") environment = Environment.from_pip_requirements(name, requirements_path) whl_url = environment.add_private_pip_wheel( workspace=self.workspace, file_path=pip_wheel_path, exist_ok=True) environment.python.conda_dependencies.add_pip_package(whl_url) return environment
def run_script_in_cloud(): ws = Workspace.from_config() experiment = Experiment(workspace=ws, name='day1-experiment-hello') config = ScriptRunConfig(source_directory="../../", script='src/azure/modelling.py', compute_target='cpu-cluster') env = Environment.from_pip_requirements(name="env", file_path="../../requirements.txt") config.run_config.environment = env run = experiment.submit(config) aml_url = run.get_portal_url() print(aml_url)
def configure_environment(workspace, wheel_file=None): # collect external requirements from requirements file environment = Environment.from_pip_requirements(name="env", file_path="requirements.txt") # add private pip wheel to blob if provided if wheel_file: private_pkg = environment.add_private_pip_wheel(workspace, file_path=wheel_file, exist_ok=True) environment.python.conda_dependencies.add_pip_package(private_pkg) # add azureml-sdk to log metrics environment.python.conda_dependencies.add_pip_package("azureml-sdk") # set docker to enabled for AmlCompute environment.docker.enabled = True print("environment successfully configured") return environment
def getEnvironment(ws: Workspace, environment_name: str, requirements_file, create_new=False): environments = Environment.list(workspace=ws) # Retrieve existing environment: if not create_new: existing_environment = None for env in environments: if env == environment_name: return environments[environment_name] # Create new environment: new_environment = Environment.from_pip_requirements( environment_name, requirements_file, ) new_environment.register(ws) return new_environment
def create_or_update_mlapp_env(workspace, requirements_path, wheel_path, env_name): """ Usage: ws = init_workspace() create_mlapp_environment( workspace=ws, requirements_path='../../../requirements.txt', wheel_path='./../../dist/mlapp-2.0.0-py3-none-any.whl', env_name='mlapp') """ # get or create environment and add requirements.txt file try: restored_env = Environment.get(workspace=workspace, name=env_name) new_env = restored_env.from_pip_requirements( name=env_name, file_path=requirements_path) except Exception as e: new_env = Environment.from_pip_requirements( name=env_name, file_path=requirements_path) # settings for environment new_env.docker.enabled = True new_env.python.user_managed_dependencies = False # add private package whl_url = Environment.add_private_pip_wheel(workspace, wheel_path, exist_ok=False) new_env.python.conda_dependencies.add_pip_package(whl_url) # build and register environment new_env = new_env.register(workspace) build_env_run = new_env.build(workspace) build_env_run.wait_for_completion(show_output=False) print(build_env_run.log_url) print(build_env_run.status)
experiment_name = "template-workflow-base" data_uri = "https://azuremlexamples.blob.core.windows.net/datasets/iris.csv" # convert to relative paths prefix = Path(__file__).parent source_dir = str(prefix.joinpath(source_dir)) environment_file = str(prefix.joinpath(environment_file)) # get workspace ws = Workspace.from_config() # create dataset ds = Dataset.File.from_files(data_uri) # create environment env = Environment.from_pip_requirements(environment_name, environment_file) # setup entry script arguments args = ["--data-dir", ds.as_mount()] # create a job configuration src = ScriptRunConfig( source_directory=source_dir, script=entry_script, arguments=args, environment=env, compute_target=compute_name, ) # run the job run = Experiment(ws, experiment_name).submit(src)
# 3. Training Step # 4. Model Registration Step # 5. Pipeline registration # 6. Submit the pipeline for execution ## Pipeline Parameters ## # We need to tell the Pipeline what it needs to learn to see! datapath = DataPath(datastore=datastore, path_on_datastore=datastorepath) data_path_pipeline_param = (PipelineParameter(name="data", default_value=datapath), DataPathComputeBinding(mode='mount')) # Configuration for data prep and training steps # dataprepEnvironment = Environment.from_pip_requirements( 'dataprepenv', 'requirements-dataprepandtraining.txt') dataprepRunConfig = RunConfiguration() dataprepRunConfig.environment = dataprepEnvironment ## Data Process Step ## # parse.py file parses the images in our data source # seer_tfrecords = PipelineData("tfrecords_set", datastore=datastore, is_directory=True) prepStep = PythonScriptStep('parse.py', source_directory='.', name='Data Preparation', compute_target=compute, arguments=[
svc_pr = ServicePrincipalAuthentication( tenant_id=svcpr_data['tenantId'], service_principal_id=svcpr_data['clientId'], service_principal_password=svcpr_data['clientSecret']) # --- get workspace, compute target, run config print("Getting workspace and compute target...") workspace = Workspace(subscription_id=azure_subscription_id, resource_group=resource_group, workspace_name=workspace_name, auth=svc_pr) compute_target = ComputeTarget(workspace=workspace, name=cpu_cluster_name) requirements_path = Path("./requirements.txt").resolve() environment = Environment.from_pip_requirements(name="SpacyEnvironment", file_path=requirements_path) run_config = RunConfiguration() run_config.environment = environment # run_config = RunConfiguration( # conda_dependencies=CondaDependencies.create( # conda_packages=['pip', 'pandas', 'scikit-learn', 'PyYAML'], # # notes: - see run_config.environment.add_private_pip_wheel() to use your own private packages, # # - you can also reference curated or custom environments here for simplification, # # see https://docs.microsoft.com/en-us/azure/machine-learning/how-to-create-your-first-pipeline for # # more details # pip_packages=["azureml-defaults", "azureml-pipeline-steps"], # ) # ) # define if docker images should be created or not
#Connecting the Workspace from azureml.core import Workspace ws = Workspace.from_config() # we can create env via several ways # 1. via conda file from azureml.core import Environment env = Environment.from_conda_specification(name = "training_env", file_path = "./conda.yml") # 2. via pip requiremnt file env.from azureml.core import Environment env = Environment.from_pip_requirements( name='env_name', file_path='requirements.txt', ) # 3. via existing conda env env = Environment.from_existing_conda_environment(name = "training_env", conda_environment_name = 'py_env') # 4. via specifying packages from azureml.core.conda_dependencies import CondaDependencies env = Environment("training_env") deps = CondaDependencies.create(conda_packages=['scikit-learn', 'pandas', 'numpy'], pip_packages=['azureml-defaults']) env.python.conda_dependencies = deps
script='AccentClassification.py', compute_target='model-training-machine', arguments=[ '--train_data_path', train_dataset.as_mount(), '--test_data_path', test_dataset.as_mount() ]) # model = Model.register(model_path = "./models/model.pt", # model_name = "accent_detection", # description = "distinguish native English accent from foreign accents", # workspace = ws) # set up pytorch environment env = Environment.from_pip_requirements( name='accent_scoring_env', file_path= '/Users/yejinseo/Desktop/azure_ai_hack/accent_scoring/requirements.txt' ) # Creates the environment inside a Docker container. env.docker.enabled = True # Specify docker steps as a string. dockerfile = r''' FROM mcr.microsoft.com/azureml/intelmpi2018.3-ubuntu16.04 RUN echo "Hello from custom container!" RUN apt-get install -y libsndfile1 RUN apt-get install -y ffmpeg RUN dpkg -L ffmpeg ''' # RUN add-apt-repository ppa:mc3man/trusty-media # RUN apt-get update # RUN apt-get install -y ffmpeg
def get_environment(workspace, name, pip_requirements=None, conda_specification=None, conda_env=None, docker_image=None, docker_file=None, override=False, inference_stack=None): """ Get an Azure ML environment from PIP or Conda. From: - pip_requirements - conda_specification - conda_env at most one can be provided. If none is provided, it is assumed that the requirements are taken care of by the user. From: - docker_image - docker_file at most one can be provided. If none is provided, the base Azure image is used. :params workspace: The Azure ML workspace to look for existing environments. :params name: Name for this environment :params pip_requirements: Path to the pip requirements file :params conda_specifidation: Path to the conda specification file :params conda_env: Name of the conda environment to use :params docker_image: Base the image off an existing docker image :params docker_file: Base the image off a Dockerfile. :params override: Create a new environment with this name, regardless of if one already exists. :params inference_stack: Add a stack that enables this environment for inference. "latest" is a valid option. Set to None to not add this. :returns: Azure ML environment or None in case of failure """ if not override: try: env = Environment.get(workspace, name) print("Existing environment found, using that") return env except: print("No environment with that name found, creating new one") # Validate at most one of pip_requirements, conda_specification, conda_env # is provided if sum([ 1 for x in [pip_requirements, conda_specification, conda_env] if x is not None ]) > 1: print("Provide at most 1 of pip_requirements, conda_specification, " "conda_env") return None # Validate that at most one of docker_image, docker_file is # provided if sum([1 for x in [docker_image, docker_file] if x is not None]) > 1: print("Provide at most 1 of docker_image, docker_file") return None if pip_requirements is not None: env = Environment.from_pip_requirements(name, pip_requirements) elif conda_specification is not None: env = Environment.from_conda_specification(name, conda_specification) elif conda_env is not None: env = Environment.from_existing_conda_environment(name, conda_env) else: env = Environment(name) env.python.user_managed_dependencies = True if docker_file is not None: env.docker.enabled = True env.docker.base_image = None env.docker.base_dockerfile = docker_file elif docker_image is not None: env.docker.enabled = True env.docker.base_image = docker_image if inference_stack is not None: env.inferencing_stack_version = inference_stack # Register environment env.register(workspace=workspace) return env