def get_environment(workspace,
env_name,
conda_dependencies,
create_new=False,
enable_docker=None,
use_gpu=False):
try:
e = ENV()
restore_env = None
environments = Environment.list(workspace=workspace)
for env in environments:
if env == env_name:
restore_env = environments[env_name]
if restore_env is None or create_new:
new_env = Environment.from_conda_specification(
env_name,
os.path.join(e.source_train_directory, conda_dependencies))
restore_env = new_env
if enable_docker:
restore_env.docker.enabled = enable_docker
restore_env.docker.base_image = DEFAULT_GPU_IMAGE if use_gpu else DEFAULT_CPU_IMAGE
restore_env.register(workspace)
if restore_env is not None:
print(f'created environment: {restore_env}')
return restore_env
except Exception as error:
print(f'Founding error: {error} occurred')
exit(1)
def get_environment(workspace: Workspace,
environment_name: str,
create_new: bool = False):
try:
e = Env()
environments = Environment.list(workspace=workspace)
restored_environment = None
# for env in environments:
# if env == environment_name:
# restored_environment = environments[environment_name]
if restored_environment is None or create_new:
new_env = Environment.from_conda_specification(
environment_name,
os.path.join(e.sources_directory_train,
"conda_dependencies.yml")) # NOQA: E501
restored_environment = new_env
restored_environment.register(workspace)
if restored_environment is not None:
print(restored_environment)
return restored_environment
except Exception as e:
print(e)
exit(1)
def get_environment(workspace: Workspace,
environment_name: str,
conda_dependencies_file: str,
create_new: bool = False,
enable_docker: bool = None,
use_gpu: bool = False):
try:
environments = Environment.list(workspace=workspace)
restored_environment = None
for env in environments:
if env == environment_name:
restored_environment = environments[environment_name]
if restored_environment is None or create_new:
new_env = Environment.from_conda_specification(
environment_name,
conda_dependencies_file,
)
restored_environment = new_env
if enable_docker is not None:
restored_environment.docker.enabled = enable_docker
restored_environment.docker.base_image = DEFAULT_GPU_IMAGE if use_gpu else DEFAULT_CPU_IMAGE
restored_environment.register(workspace)
if restored_environment is not None:
print(restored_environment)
return restored_environment
except Exception as e:
print(e)
exit(1)
def get_environment(workspace: Workspace,
environment_name: str,
conda_dependencies_file: str,
create_new: bool = False,
enable_docker: bool = None,
use_gpu: bool = False):
try:
e = Env()
environments = Environment.list(workspace=workspace)
restored_environment = None
for env in environments:
if env == environment_name:
restored_environment = environments[environment_name]
if restored_environment is None or create_new:
new_env = Environment.from_conda_specification(
environment_name,
os.path.join(e.sources_directory_train,
conda_dependencies_file), # NOQA: E501
) # NOQA: E501
restored_environment = new_env
if enable_docker is not None:
restored_environment.docker.enabled = enable_docker
restored_environment.docker.base_image = DEFAULT_GPU_IMAGE if use_gpu else DEFAULT_CPU_IMAGE # NOQA: E501
restored_environment.register(workspace)
if restored_environment is not None:
print(restored_environment)
return restored_environment
except Exception:
traceback.print_exc()
exit(1)
def get_environment(workspace: Workspace,
environment_name: str,
conda_dependencies_file: str,
create_new: bool = False,
enable_docker: bool = None,
use_gpu: bool = False):
try:
e = Env()
environments = Environment.list(workspace=workspace)
restored_environment = None
for env in environments:
if env == environment_name:
restored_environment = environments[environment_name]
if restored_environment is None or create_new:
new_env = Environment.from_conda_specification(
environment_name,
os.path.join(e.sources_directory_train,
conda_dependencies_file), # NOQA: E501
) # NOQA: E501
restored_environment = new_env
if enable_docker is not None:
DockerConfiguration(use_docker=True)
if restored_environment is not None:
print(restored_environment)
return restored_environment
except Exception as e:
print(e)
exit(1)
def show_available_environment(workspace):
list_env = Environment.list(workspace=workspace)
for env in list_env:
if env.startswith("AzureML"):
print("Name", env)
print(
"packages",
list_env[env].python.conda_dependencies.serialize_to_string())
def display_mlapp_environments(workspace, name='mlapp'):
envs = Environment.list(workspace=workspace)
for env in envs:
if env.startswith(name):
print("Name", env)
print("packages",
envs[env].python.conda_dependencies.serialize_to_string())
def show_list_of_all_environments():
try:
ws = _establish_connection_to_aml_workspace()
except Exception as e:
raise e
try:
env_list = Environment.list(ws)
for key in env_list:
print(key)
except Exception as e:
raise e
def create_conda_environment(workspace, name, conda_dependencies,
pip_dependencies):
"""
Create an environment or retrieve it by its name from workspace
Pip installs Python packages whereas conda installs packages which may contain software written in any language.
e.g. TensorFlow, Scikit-Learn -> Conda, Matplotlib -> pip
"""
if name in Environment.list(workspace):
env = Environment.get(workspace=workspace, name=name)
print("The environment '{}' already existed for the workspace".format(
name))
else:
env = Environment(name=name)
env.docker.enabled = True
env.python.conda_dependencies = CondaDependencies.create(
conda_packages=conda_dependencies,
pip_packages=pip_dependencies,
)
env.register(workspace=workspace)
return env
def getEnvironment(ws: Workspace,
environment_name: str,
requirements_file,
create_new=False):
environments = Environment.list(workspace=ws)
# Retrieve existing environment:
if not create_new:
existing_environment = None
for env in environments:
if env == environment_name:
return environments[environment_name]
# Create new environment:
new_environment = Environment.from_pip_requirements(
environment_name,
requirements_file,
)
new_environment.register(ws)
return new_environment
env.docker.base_image='my-base-image'
env.docker.base_image_registry='myregistry.azurecr.io/myimage'
#override how AZ auto handles pkg dependencies etc
env.python.user_managed_dependencies=True
env.python.interpreter_path = '/opt/miniconda/bin/python'
#register
env.register(workspace=ws)
#view registered
from azureml.core import Environment
env_names = Environment.list(workspace=ws)
for env_name in env_names:
print('Name:',env_name)
#retrieve enviro
from azureml.core import Environment
from azureml.train.estimator import Estimator
training_env = Environment.get(workspace=ws, name='training_environment')
estimator = Estimator(source_directory='experiment_folder'
entry_script='training_script.py',
compute_target='local',
environment_definition=training_env)
#make Managed COmpute target
def get_environment(
workspace: Workspace,
environment_name: str,
conda_dependencies_file: str = None,
create_new: bool = False,
enable_docker: bool = None,
docker_image: str = None,
dockerfile: str = None,
use_gpu: bool = False,
):
try:
e = Env()
environments = Environment.list(workspace=workspace)
restored_environment = None
for env in environments:
if env == environment_name:
restored_environment = environments[environment_name]
if restored_environment is None or create_new:
# Environment has to be created
if conda_dependencies_file is not None:
new_env = Environment.from_conda_specification(
environment_name,
os.path.join(e.sources_directory_train, conda_dependencies_file), # NOQA: E501
) # NOQA: E501
restored_environment = new_env
else:
restored_environment = Environment(environment_name)
if enable_docker is not None:
restored_environment.docker.enabled = enable_docker
if docker_image is not None:
restored_environment.docker.base_image = docker_image
# In case of own image
# don't append AML managed dependencies
restored_environment.python.\
user_managed_dependencies = True
elif dockerfile is not None:
# Alternatively, load from a file.
with open(dockerfile, "r") as f:
dockerfile = f.read()
restored_environment.docker.\
base_dockerfile = dockerfile
# In case of own Dockerfile
# don't append AML managed dependencies
restored_environment.python.\
user_managed_dependencies = True
else:
restored_environment.docker.\
base_image = DEFAULT_GPU_IMAGE if use_gpu else DEFAULT_CPU_IMAGE # NOQA: E501
restored_environment.register(workspace)
if restored_environment is not None:
print(restored_environment)
return restored_environment
except Exception as e:
print(e)
exit(1)
# 4. via specifying packages
from azureml.core.conda_dependencies import CondaDependencies
env = Environment("training_env")
deps = CondaDependencies.create(conda_packages=['scikit-learn', 'pandas', 'numpy'],
pip_packages=['azureml-defaults'])
env.python.conda_dependencies = deps
# Registering the environment to the workspace
env.register(workspace = ws)
# get list of all the environments in the workspace
for env_name in Environment.list(workspace = ws):
print('Name:', env_name)
#for using in the script
from azureml.core import ScriptRunConfig, Environment
training_env = Environment.get(workspace = ws, name = "training_env")
script_config = ScriptRunConfig(source_directory = "experiments_directory",
script = "script.py",
environment = training_env)
# In[4]:
instance = ComputeTarget(workspace=workspace, name='gpu-compute-low')
# In[5]:
dataset = Dataset.get_by_name(workspace, name='recursionbio')
# In[6]:
dataset
# In[7]:
env_list = Environment.list(workspace)
# In[8]:
env_list.keys()
# In[9]:
tf_env = Environment.get(workspace=workspace,
name='AzureML-TensorFlow-2.3-GPU')
tf_env = tf_env.clone(new_name='recbio-tf-2.3-efficientnet')
# In[10]:
tf_env.python.conda_dependencies.add_conda_package('scikit-learn')
tf_env.python.conda_dependencies.add_conda_package('scipy')
# COMMAND ----------
# MAGIC %md
# MAGIC - 환경 구축
# MAGIC - 기존에 구성된 환경이나 직접 설정한 환경을 구성하면 해당 환경을 재사용 가능 하며 재사용시 ARC에 있는 이미지도 재 사용이 가능함.
# COMMAND ----------
from azureml.core import Workspace, Environment
ws = Workspace.from_config()
#env = Environment.get(workspace=ws, name="toone_env_databricks")
envs = Environment.list(workspace=ws)
for env in envs:
if env.startswith("AzureML"):
print("Name",env)
print("packages", envs[env].python.conda_dependencies.serialize_to_string())
# COMMAND ----------
# MAGIC %md
# MAGIC - 수동으로 환경 생성
# COMMAND ----------
