def get_environment(name_suffix="_training"):
# Load the JSON settings file
print("Loading settings")
with open(os.path.join("aml_config", "settings.json")) as f:
settings = json.load(f)
env_settings = settings["environment"]
env_name = settings["experiment"]["name"] + name_suffix
# Create Dependencies
print("Defining Conda Dependencies")
conda_dep = CondaDependencies().create(
pip_indexurl=None,
pip_packages=env_settings["pip_packages"],
conda_packages=env_settings["conda_packages"],
python_version=env_settings["python_version"],
pin_sdk_version=env_settings["pin_sdk_version"])
conda_dep.save(path=env_settings["dependencies_config"]["path"])
# Create Environment and setting parameters
print("Creating Environment")
env = Environment(name=env_name)
env.python.conda_dependencies = conda_dep
env.environment_variables = env_settings["env_variables"]
if env_settings["user_managed_dependencies"]:
print("Using existing user-managed Python environment for run")
env.user_managed_dependencies = env_settings[
"user_managed_dependencies"]
elif env_settings["docker"]["enabled"]:
print(
"Using Docker run with system-built conda environment based on dependency specification"
)
env.docker.enabled = env_settings["docker"]["enabled"]
env.docker.gpu_support = env_settings["docker"]["gpu_support"]
env.docker.arguments = env_settings["docker"]["arguments"]
env.docker.shared_volumes = env_settings["docker"]["shared_volumes"]
env.docker.shm_size = env_settings["docker"]["shm_size"]
if env_settings["docker"]["gpu_support"] and env_settings["docker"][
"mpi_image"]:
env.docker.base_image = azureml.core.runconfig.MPI_GPU_IMAGE
elif env_settings["docker"]["gpu_support"]:
env.docker.base_image = azureml.core.runconfig.DEFAULT_GPU_IMAGE
elif env_settings["docker"]["mpi_image"]:
env.docker.base_image = azureml.core.runconfig.MPI_CPU_IMAGE
env.docker.base_image = env_settings["docker"]["base_image"]
env.docker.base_image_registry.address = env_settings["docker"][
"base_image_registry"]["address"]
env.docker.base_image_registry.username = env_settings["docker"][
"base_image_registry"]["username"]
env.docker.base_image_registry.password = env_settings["docker"][
"base_image_registry"]["password"]
else:
print(
"Using system-build conda environment based on dependency specification"
)
env.docker.enabled = False
return env
def get_conda_depencies():
conda_dep = CondaDependencies()
conda_dep.set_python_version("3.6.2")
with open('compute_pip_requirements.txt') as f:
for requirement in f:
conda_dep.add_pip_package(str(requirement))
with open('compute_conda_requirements.txt') as f:
for requirement in f:
conda_dep.add_conda_package(str(requirement))
return conda_dep
def dep_from_pkg_list(pkg_list):
"""Get conda dependencies from list of packages
Parameters
----------
pkg_list : list
list of conda packages
Returns
-------
CondaDependencies
collection of conda dependencies
"""
return CondaDependencies().create(conda_packages=pkg_list)
# ***************** 1. CREATE PythonScriptStep1: DATA PREPARATION **************************
# ...which needs a RunConfiguration object that specifies the right Conda/Pip dependencies. So we proceed as follows in 3 steps:
# 1.A) Create PipelineData Object datapreparation_output
from azureml.pipeline.core import PipelineData
is_directory = False # it's a file where we save the prepared dataframe
default_datastore = ws.get_default_datastore()
datapreparation_output = PipelineData('datapreparation_output',
datastore=default_datastore,
is_directory=is_directory)
# 1.B) Create the dependency object with mlextend package https://docs.microsoft.com/en-us/python/api/azureml-core/azureml.core.conda_dependencies.condadependencies?view=azure-ml-py:
from azureml.core.environment import CondaDependencies
conda_dep_prep = CondaDependencies()
conda_dep_prep.add_pip_package(
"mlxtend==0.17.2") # or conda_dep.add_conda_package("mlxtend==0.17.2")
# 1.C) Create the RunConfiguration object:
from azureml.core import RunConfiguration
run_config_prep = RunConfiguration(conda_dependencies=conda_dep_prep)
# 1.D) Create the PythonScriptStep
from azureml.pipeline.steps import PythonScriptStep
data_preparation_step = PythonScriptStep(
name="1: Data preparation",
script_name="1-data_preparation.py",
compute_target=compute_target,
runconfig=run_config_prep,
arguments=[
# Can poll for a minimum number of nodes and for a specific timeout.
# If no min node count is provided it will use the scale settings for the cluster
compute_target.wait_for_completion(show_output=True,
min_node_count=None,
timeout_in_minutes=40)
# For a more detailed view of current cluster status, use the 'status' property
print(compute_target.status.serialize())
project_folder = 'source'
# Composer les étapes de votre pipeline
from azureml.core.environment import CondaDependencies
# myenv = Environment(name="myenv")
conda_dep = CondaDependencies()
conda_dep.add_conda_package("scikit-learn")
conda_dep.add_conda_package("joblib")
conda_dep.add_conda_package("numpy")
conda_dep.add_conda_package("pandas")
conf = RunConfiguration(script="train.py", conda_dependencies=conda_dep)
from azureml.pipeline.steps import PythonScriptStep
trainStep = PythonScriptStep(
script_name="train.py",
arguments=["--input_data", blob_input_data, "--output", output_data1],
inputs=[blob_input_data],
outputs=[output_data1],
compute_target=compute_target,
source_directory=project_folder,
runconfig=conf,
workspace = Workspace.create(name=conf.WORKSPACE_NAME,
location=conf.WORKSPACE_LOCATION,
resource_group=conf.RESOURCE_GROUP,
subscription_id=conf.SUBSCRIPTION_ID,
auth=svc_pr,
exist_ok=True)
# Get the model registered in your Azure ML Workspace
model = Model.list(workspace, conf.MODEL_NAME)[0]
# Create the environment to manage the dependencies
env = Environment(name="env-local-deploy")
# Here we will use a .yml file to control the dependencies
conda_dep = CondaDependencies(conda_dependencies_file_path='conda_env.yml')
# Adds dependencies to PythonSection of myenv
env.python.conda_dependencies = conda_dep
# Our inference config with the score.py file
inference_config = InferenceConfig(entry_script="score.py", environment=env)
# Create a local deployment, using port 8890 for the web service endpoint
deployment_config = LocalWebservice.deploy_configuration(port=conf.PORT)
# Deploy the service
service = Model.deploy(workspace, conf.ENDPOINT_NAME, [model],
inference_config, deployment_config)
# Wait for the deployment to complete
cat.azureml / config.json
# In[5]:
get_ipython().run_cell_magic(
'writefile', 'summarizer_service.py',
'\nimport re\nimport nltk\nimport unicodedata\nfrom gensim.summarization import summarize, keywords\n\ndef clean_and_parse_document(document):\n if isinstance(document, str):\n document = document\n elif isinstance(document, unicode):\n return unicodedata.normalize(\'NFKD\', document).encode(\'ascii\', \'ignore\')\n else:\n raise ValueError("Document is not string or unicode.")\n document = document.strip()\n sentences = nltk.sent_tokenize(document)\n sentences = [sentence.strip() for sentence in sentences]\n return sentences\n\ndef summarize_text(text, summary_ratio=None, word_count=30):\n sentences = clean_and_parse_document(text)\n cleaned_text = \' \'.join(sentences)\n summary = summarize(cleaned_text, split=True, ratio=summary_ratio, word_count=word_count)\n return summary \n\ndef init(): \n nltk.download(\'all\')\n return\n\ndef run(input_str):\n try:\n return summarize_text(input_str)\n except Exception as e:\n return (str(e))'
)
# In[7]:
from azureml.core import Environment
from azureml.core.environment import CondaDependencies
myenv = Environment(name="myenv")
conda_dep = CondaDependencies()
conda_dep.add_pip_package("gensim")
conda_dep.add_pip_package("nltk")
myenv.python.conda_dependencies = conda_dep
# In[8]:
myenv.register(workspace=ws)
# In[9]:
from azureml.core.model import InferenceConfig
from azureml.core.webservice import AciWebservice
inference_config = InferenceConfig(entry_script='summarizer_service.py',
environment=myenv)
filename = 'dummy.pkl'
pickle.dump(model, open(filename, 'wb'))
# register the dummy model
ws = Workspace.from_config()
registration_params = {
'model_path': "dummy.pkl",
'model_name': "dummy-model",
'description': "mock test deployment",
'workspace': ws
}
model = register_model(**registration_params)
myenv = Environment(name='my_env')
myenv.get(workspace=ws, name='ls-ds-ml-env')
conda_dep = CondaDependencies()
conda_dep.add_pip_package("azureml-defaults==1.3.0")
conda_dep.add_pip_package("joblib")
conda_dep.add_pip_package("json")
myenv.python.conda_dependencies = conda_dep
inference_config = InferenceConfig(entry_script="src/dummy_score.py",
environment=myenv)
aci_config = AciWebservice.deploy_configuration(cpu_cores=1, memory_gb=1)
service_name = 'dummy-service'
try:
service = Model.deploy(ws,
service_name, [model],
inference_config,
deployment_config=aci_config)
service.wait_for_deployment(True)
)
kv = Keyvault(ws)
# Usually, the cluster already exists, so we just fetch
compute_target = next(
(m for m in ComputeTarget.list(ws) if m.name == compute["name"]), None)
# Env for use case
aml = RCranPackage()
aml.name = "azuremlsdk"
aml.version = "1.10.0"
cd = CondaDependencies.create(
conda_packages=["pandas", "numpy", "matplotlib"],
pip_packages=["azureml-mlflow==1.17.0", "azureml-defaults==1.17.0"],
)
rc = RunConfiguration()
rc.framework = "R"
rc.environment.r = RSection()
# rc.environment.r.cran_packages = [aml]
rc.environment.docker.enabled = True
py_rc = RunConfiguration()
py_rc.framework = "Python"
py_rc.environment.python.conda_dependencies = cd
sql_datastore = Datastore.register_azure_sql_database(
workspace=ws,
datastore_name="modelling_db",
from azureml.core import Workspace, Experiment, ScriptRunConfig
ws = Workspace.from_config()
experiment = Experiment(ws, "training-exp-02")
from azureml.core import Environment
from azureml.core.environment import CondaDependencies
#creating Custom Environment
my_env = Environment(name="MyEnvironment2")
#creating the Dependencies
my_env_dep = CondaDependencies.create(conda_packages=['scikit-learn'])
my_env.python.conda_dependencies = my_env_dep
#registering the Environment
my_env.register(ws)
script_config = ScriptRunConfig(source_directory=".",
script="02-training_script.py",
environment=my_env)
new_run = experiment.submit(config=script_config)
new_run.wait_for_completion()
for compute_name in ws.compute_targets:
compute = ws.compute_targets[compute_name]
print(compute.name, ":", compute.type)
#ws.name
#ws.location
#ws.subscription_id
#ws.resource_group
from azureml.core import Enviornment
env = Enviornment.get(workspace=ws, nmae="qwer")
env.get_image_details(ws)
from azureml.core.environment import CondaDependencies
myenv = Environment(name="myenv")
conda_dep = CondaDependencies()
conda_dep.add_conda_package("scikit-learn")
from azureml.core import Dataset
dataset = Dataset.Tabular.from_delimited_files(path = [(datastore, 'train-dataset/tabular/iris.csv')])
# preview the first 3 rows of the dataset
dataset.take(3).to_pandas_dataframe()
from azureml.core.compute import ComputeTarget, AmlCompute
from azureml.core.compute_target import ComputeTargetException
# Choose a name for your CPU cluster
cpu_cluster_name = "cpu-cluster"
# Verify that cluster does not exist already
try: