def create_env_from_requirements(file_path='requirements.txt',
name='mlapp',
endpoint=False):
env = Environment.from_pip_requirements(name=name, file_path=file_path)
# Enable Docker
env.docker.enabled = True
# Set Docker base image to the default CPU-based image
path_to_dockerfile = os.path.join(os.getcwd(), 'deployment', 'Dockerfile')
if os.path.exists(path_to_dockerfile):
# Set custom Docker image
env.docker.base_image = None
env.docker.base_dockerfile = path_to_dockerfile
else:
# Set Docker base image to the default CPU-based image
env.docker.base_image = DEFAULT_CPU_IMAGE
# Use conda_dependencies.yml to create a conda environment in the Docker image for execution
env.python.user_managed_dependencies = False
if endpoint:
env.inferencing_stack_version = 'latest'
return env
def transformers_environment(use_gpu=True):
"""Prepares Azure ML Environment with transformers library.
Note: We install transformers library from source. See requirements file for
full list of dependencies.
Args:
use_gpu (bool): If true, Azure ML will use gpu-enabled docker image
as base.
Return:
Azure ML Environment with huggingface libraries needed to perform GLUE
finetuning task.
"""
pip_requirements_path = str(
Path(__file__).parent.joinpath("requirements.txt"))
print(f"Create Azure ML Environment from {pip_requirements_path}")
if use_gpu:
env_name = "transformers-gpu"
env = Environment.from_pip_requirements(
name=env_name,
file_path=pip_requirements_path,
)
env.docker.base_image = (
"mcr.microsoft.com/azureml/intelmpi2018.3-cuda10.0-cudnn7-ubuntu16.04"
)
else:
env_name = "transformers-cpu"
env = Environment.from_pip_requirements(
name=env_name,
file_path=pip_requirements_path,
)
return env
def get_environment(self, config):
name = config.get("name")
pip_wheel_path = config.get("pip_wheel_path")
requirements_path = config.get("requirements_path")
environment = Environment.from_pip_requirements(name, requirements_path)
whl_url = environment.add_private_pip_wheel(
workspace=self.workspace,
file_path=pip_wheel_path,
exist_ok=True)
environment.python.conda_dependencies.add_pip_package(whl_url)
return environment
def run_script_in_cloud():
ws = Workspace.from_config()
experiment = Experiment(workspace=ws, name='day1-experiment-hello')
config = ScriptRunConfig(source_directory="../../",
script='src/azure/modelling.py',
compute_target='cpu-cluster')
env = Environment.from_pip_requirements(name="env",
file_path="../../requirements.txt")
config.run_config.environment = env
run = experiment.submit(config)
aml_url = run.get_portal_url()
print(aml_url)
def configure_environment(workspace, wheel_file=None):
# collect external requirements from requirements file
environment = Environment.from_pip_requirements(name="env", file_path="requirements.txt")
# add private pip wheel to blob if provided
if wheel_file:
private_pkg = environment.add_private_pip_wheel(workspace, file_path=wheel_file,
exist_ok=True)
environment.python.conda_dependencies.add_pip_package(private_pkg)
# add azureml-sdk to log metrics
environment.python.conda_dependencies.add_pip_package("azureml-sdk")
# set docker to enabled for AmlCompute
environment.docker.enabled = True
print("environment successfully configured")
return environment
def getEnvironment(ws: Workspace,
environment_name: str,
requirements_file,
create_new=False):
environments = Environment.list(workspace=ws)
# Retrieve existing environment:
if not create_new:
existing_environment = None
for env in environments:
if env == environment_name:
return environments[environment_name]
# Create new environment:
new_environment = Environment.from_pip_requirements(
environment_name,
requirements_file,
)
new_environment.register(ws)
return new_environment
def create_or_update_mlapp_env(workspace, requirements_path, wheel_path,
env_name):
"""
Usage:
ws = init_workspace()
create_mlapp_environment(
workspace=ws,
requirements_path='../../../requirements.txt',
wheel_path='./../../dist/mlapp-2.0.0-py3-none-any.whl',
env_name='mlapp')
"""
# get or create environment and add requirements.txt file
try:
restored_env = Environment.get(workspace=workspace, name=env_name)
new_env = restored_env.from_pip_requirements(
name=env_name, file_path=requirements_path)
except Exception as e:
new_env = Environment.from_pip_requirements(
name=env_name, file_path=requirements_path)
# settings for environment
new_env.docker.enabled = True
new_env.python.user_managed_dependencies = False
# add private package
whl_url = Environment.add_private_pip_wheel(workspace,
wheel_path,
exist_ok=False)
new_env.python.conda_dependencies.add_pip_package(whl_url)
# build and register environment
new_env = new_env.register(workspace)
build_env_run = new_env.build(workspace)
build_env_run.wait_for_completion(show_output=False)
print(build_env_run.log_url)
print(build_env_run.status)
experiment_name = "template-workflow-base"
data_uri = "https://azuremlexamples.blob.core.windows.net/datasets/iris.csv"
# convert to relative paths
prefix = Path(__file__).parent
source_dir = str(prefix.joinpath(source_dir))
environment_file = str(prefix.joinpath(environment_file))
# get workspace
ws = Workspace.from_config()
# create dataset
ds = Dataset.File.from_files(data_uri)
# create environment
env = Environment.from_pip_requirements(environment_name, environment_file)
# setup entry script arguments
args = ["--data-dir", ds.as_mount()]
# create a job configuration
src = ScriptRunConfig(
source_directory=source_dir,
script=entry_script,
arguments=args,
environment=env,
compute_target=compute_name,
)
# run the job
run = Experiment(ws, experiment_name).submit(src)
# 3. Training Step
# 4. Model Registration Step
# 5. Pipeline registration
# 6. Submit the pipeline for execution
## Pipeline Parameters ##
# We need to tell the Pipeline what it needs to learn to see!
datapath = DataPath(datastore=datastore, path_on_datastore=datastorepath)
data_path_pipeline_param = (PipelineParameter(name="data",
default_value=datapath),
DataPathComputeBinding(mode='mount'))
# Configuration for data prep and training steps #
dataprepEnvironment = Environment.from_pip_requirements(
'dataprepenv', 'requirements-dataprepandtraining.txt')
dataprepRunConfig = RunConfiguration()
dataprepRunConfig.environment = dataprepEnvironment
## Data Process Step ##
# parse.py file parses the images in our data source #
seer_tfrecords = PipelineData("tfrecords_set",
datastore=datastore,
is_directory=True)
prepStep = PythonScriptStep('parse.py',
source_directory='.',
name='Data Preparation',
compute_target=compute,
arguments=[
svc_pr = ServicePrincipalAuthentication(
tenant_id=svcpr_data['tenantId'],
service_principal_id=svcpr_data['clientId'],
service_principal_password=svcpr_data['clientSecret'])
# --- get workspace, compute target, run config
print("Getting workspace and compute target...")
workspace = Workspace(subscription_id=azure_subscription_id,
resource_group=resource_group,
workspace_name=workspace_name,
auth=svc_pr)
compute_target = ComputeTarget(workspace=workspace, name=cpu_cluster_name)
requirements_path = Path("./requirements.txt").resolve()
environment = Environment.from_pip_requirements(name="SpacyEnvironment",
file_path=requirements_path)
run_config = RunConfiguration()
run_config.environment = environment
# run_config = RunConfiguration(
# conda_dependencies=CondaDependencies.create(
# conda_packages=['pip', 'pandas', 'scikit-learn', 'PyYAML'],
# # notes: - see run_config.environment.add_private_pip_wheel() to use your own private packages,
# # - you can also reference curated or custom environments here for simplification,
# # see https://docs.microsoft.com/en-us/azure/machine-learning/how-to-create-your-first-pipeline for
# # more details
# pip_packages=["azureml-defaults", "azureml-pipeline-steps"],
# )
# )
# define if docker images should be created or not
#Connecting the Workspace
from azureml.core import Workspace
ws = Workspace.from_config()
# we can create env via several ways
# 1. via conda file
from azureml.core import Environment
env = Environment.from_conda_specification(name = "training_env", file_path = "./conda.yml")
# 2. via pip requiremnt file
env.from azureml.core import Environment
env = Environment.from_pip_requirements(
name='env_name',
file_path='requirements.txt',
)
# 3. via existing conda env
env = Environment.from_existing_conda_environment(name = "training_env", conda_environment_name = 'py_env')
# 4. via specifying packages
from azureml.core.conda_dependencies import CondaDependencies
env = Environment("training_env")
deps = CondaDependencies.create(conda_packages=['scikit-learn', 'pandas', 'numpy'],
pip_packages=['azureml-defaults'])
env.python.conda_dependencies = deps
script='AccentClassification.py',
compute_target='model-training-machine',
arguments=[
'--train_data_path',
train_dataset.as_mount(), '--test_data_path',
test_dataset.as_mount()
])
# model = Model.register(model_path = "./models/model.pt",
# model_name = "accent_detection",
# description = "distinguish native English accent from foreign accents",
# workspace = ws)
# set up pytorch environment
env = Environment.from_pip_requirements(
name='accent_scoring_env',
file_path=
'/Users/yejinseo/Desktop/azure_ai_hack/accent_scoring/requirements.txt'
)
# Creates the environment inside a Docker container.
env.docker.enabled = True
# Specify docker steps as a string.
dockerfile = r'''
FROM mcr.microsoft.com/azureml/intelmpi2018.3-ubuntu16.04
RUN echo "Hello from custom container!"
RUN apt-get install -y libsndfile1
RUN apt-get install -y ffmpeg
RUN dpkg -L ffmpeg
'''
# RUN add-apt-repository ppa:mc3man/trusty-media
# RUN apt-get update
# RUN apt-get install -y ffmpeg
def get_environment(workspace,
name,
pip_requirements=None,
conda_specification=None,
conda_env=None,
docker_image=None,
docker_file=None,
override=False,
inference_stack=None):
"""
Get an Azure ML environment from PIP or Conda. From:
- pip_requirements
- conda_specification
- conda_env
at most one can be provided. If none is provided, it is assumed that the
requirements are taken care of by the user.
From:
- docker_image
- docker_file
at most one can be provided. If none is provided, the base Azure image is
used.
:params workspace: The Azure ML workspace to look for existing
environments.
:params name: Name for this environment
:params pip_requirements: Path to the pip requirements file
:params conda_specifidation: Path to the conda specification file
:params conda_env: Name of the conda environment to use
:params docker_image: Base the image off an existing docker image
:params docker_file: Base the image off a Dockerfile.
:params override: Create a new environment with this name,
regardless of if one already exists.
:params inference_stack: Add a stack that enables this environment
for inference. "latest" is a valid option.
Set to None to not add this.
:returns: Azure ML environment or None in case of
failure
"""
if not override:
try:
env = Environment.get(workspace, name)
print("Existing environment found, using that")
return env
except:
print("No environment with that name found, creating new one")
# Validate at most one of pip_requirements, conda_specification, conda_env
# is provided
if sum([
1 for x in [pip_requirements, conda_specification, conda_env]
if x is not None
]) > 1:
print("Provide at most 1 of pip_requirements, conda_specification, "
"conda_env")
return None
# Validate that at most one of docker_image, docker_file is
# provided
if sum([1 for x in [docker_image, docker_file] if x is not None]) > 1:
print("Provide at most 1 of docker_image, docker_file")
return None
if pip_requirements is not None:
env = Environment.from_pip_requirements(name, pip_requirements)
elif conda_specification is not None:
env = Environment.from_conda_specification(name, conda_specification)
elif conda_env is not None:
env = Environment.from_existing_conda_environment(name, conda_env)
else:
env = Environment(name)
env.python.user_managed_dependencies = True
if docker_file is not None:
env.docker.enabled = True
env.docker.base_image = None
env.docker.base_dockerfile = docker_file
elif docker_image is not None:
env.docker.enabled = True
env.docker.base_image = docker_image
if inference_stack is not None:
env.inferencing_stack_version = inference_stack
# Register environment
env.register(workspace=workspace)
return env
