def get_environment( ws, environment_name, docker_image="todrabas/aml_rapids:latest", python_interpreter="/opt/conda/envs/rapids/bin/python", conda_packages=["matplotlib"], ): if environment_name not in ws.environments: env = Environment(name=environment_name) env.docker.enabled = True env.docker.base_image = docker_image env.python.interpreter_path = python_interpreter env.python.user_managed_dependencies = True conda_dep = CondaDependencies() for conda_package in conda_packages: conda_dep.add_conda_package(conda_package) env.python.conda_dependencies = conda_dep env.register(workspace=ws) else: env = ws.environments[environment_name] return env
def get_run_cfg(ws, pip_packages, conda_packages, ext_wheels, gpu=True): ''' get_run_cfg - Retrieves the AMLS run configuration. :returns: AMLS run configuration :rtype: RunConfiguration object ''' conda_dep = CondaDependencies() for pip_package in pip_packages: conda_dep.add_pip_package(pip_package) for conda_package in conda_packages: conda_dep.add_conda_package(conda_package) for whl_path in ext_wheels: whl_url = Environment.add_private_pip_wheel(workspace=ws, file_path=whl_path, exist_ok=True) conda_dep.add_pip_package(whl_url) run_cfg = RunConfiguration(conda_dependencies=conda_dep) run_cfg.environment.docker.enabled = True run_cfg.environment.docker.gpu_support = gpu if gpu: run_cfg.environment.docker.base_image = DEFAULT_GPU_IMAGE else: run_cfg.environment.docker.base_image = DEFAULT_CPU_IMAGE run_cfg.environment.spark.precache_packages = False return run_cfg
def createOrGetEnvironment(ws, login_config, app_config): environment_name = login_config["aml_compute"]["environment_name"] python_interpreter = login_config["aml_compute"]["python_interpreter"] conda_packages = login_config["aml_compute"]["conda_packages"] ### CREATE OR RETRIEVE THE ENVIRONMENT if environment_name not in ws.environments: logger.info(f"Creating {environment_name} environment...") env = Environment(name=environment_name) env.docker.enabled = login_config["aml_compute"]["docker_enabled"] env.docker.base_image = None env.docker.base_dockerfile = f'FROM {app_config["base_dockerfile"]}' env.python.interpreter_path = python_interpreter env.python.user_managed_dependencies = True conda_dep = CondaDependencies() for conda_package in conda_packages: conda_dep.add_conda_package(conda_package) env.python.conda_dependencies = conda_dep env.register(workspace=ws) evn = env else: logger.info(f" Environment {environment_name} found...") env = ws.environments[environment_name] return env
def main(): ws = Workspace.from_config() conda = CondaDependencies() conda.add_conda_package("python==3.5") conda.add_pip_package("h5py==2.8.0") conda.add_pip_package("html5lib==1.0.1") conda.add_pip_package("keras==2.2.0") conda.add_pip_package("Keras-Applications==1.0.2") conda.add_pip_package("Keras-Preprocessing==1.0.1") conda.add_pip_package("matplotlib==2.2.2") conda.add_pip_package("numpy==1.14.5") conda.add_pip_package("opencv-python==3.3.0.9") conda.add_pip_package("pandas==0.23.3") conda.add_pip_package("Pillow==5.2.0") conda.add_pip_package("requests==2.19.1") conda.add_pip_package("scikit-image==0.14.0") conda.add_pip_package("scikit-learn==0.19.2") conda.add_pip_package("scipy==1.1.0") conda.add_pip_package("sklearn==0.0") conda.add_pip_package("tensorflow==1.9.0") conda.add_pip_package("urllib3==1.23") conda.add_pip_package("azureml-sdk") with open("environment.yml", "w") as f: f.write(conda.serialize_to_string()) with open("environment.yml", "r") as f: print(f.read()) image_config = ContainerImage.image_configuration( execution_script="score.py", runtime="python", conda_file="environment.yml", docker_file="Dockerfile", dependencies=DEPENDENCIES) webservices = ws.webservices(compute_type='ACI') image = ContainerImage.create(name="ai-bootcamp", models=[], image_config=image_config, workspace=ws) image.wait_for_creation(show_output=True) webservices_list = [] for key in webservices: webservices_list.append(key) service_name = webservices_list[0] aciwebservice = AciWebservice(ws, service_name) aciwebservice.update(image=image)
def create_yaml_file(): myenv = CondaDependencies() myenv.add_conda_package("scikit-learn") myenv.add_conda_package("pandas") with open("myenv.yml", "w") as f: f.write(myenv.serialize_to_string()) with open("myenv.yml", "r") as f: print(f.read())
def conda_dependencies(self): """ Get module conda dependencies :return: CondaDependencies instance """ cd = CondaDependencies() for c in self._get_value('CondaDependencies/CondaChannels'): cd.add_channel(c) for c in self._get_value('CondaDependencies/CondaPackages'): cd.add_conda_package(c) for p in self._get_value('CondaDependencies/PipPackages'): cd.add_pip_package(p) for p in self._get_value('CondaDependencies/PipOptions'): cd.set_pip_option(p) return cd
def get_config(entry_script): # Create the environment env = Environment(name="tensorflow_env") conda_dep = CondaDependencies() # Define the packages needed by the model and scripts conda_dep.add_conda_package("tensorflow") # You must list azureml-defaults as a pip dependency conda_dep.add_pip_package("azureml-defaults") conda_dep.add_pip_package("keras") conda_dep.add_pip_package("pandas") # Adds dependencies to PythonSection of myenv env.python.conda_dependencies = conda_dep inference_config = InferenceConfig(entry_script=entry_script, environment=env) print('Configuração do Endpoint retornada') return inference_config
def create_env(): myenv = CondaDependencies() myenv.add_conda_package("pytorch") myenv.add_conda_package("numpy") myenv.add_conda_package("torchvision=0.4.1") with open("./myenv.yml", "w") as f: f.write(myenv.serialize_to_string()) return "./myenv.yml"
from azureml.core import Workspace, Environment from azureml.core.conda_dependencies import CondaDependencies ws = Workspace.from_config() # Start from minimal my_env = Environment.get(workspace=ws, name="AzureML-Minimal") # Rename the env my_env.name = "gensim-environment" # Add dependencies conda_deps = CondaDependencies() conda_deps.add_conda_package("gensim") conda_deps.add_conda_package("nltk") my_env.python.conda_dependencies = conda_deps # Register the env my_env.register(workspace=ws)
# Shows output of the run on stdout. run.wait_for_completion(show_output=True) # Editing a run configuration property on-fly. run_config = RunConfiguration.load(project_object=project, run_config_name="local") # Use a new conda environment that is to be created from the conda_dependencies.yml file run_config.environment.python.user_managed_dependencies = False # Automatically create the conda environment before the run run_config.prepare_environment = True from azureml.core.conda_dependencies import CondaDependencies cd = CondaDependencies() cd.add_conda_package('scikit-learn') cd.save_to_file(project_dir=project_folder, file_name='conda_dependencies.yml') print() print('##################################################') print("Submitting {} for a local conda run...".format(train_script)) print('##################################################') print() run = Run.submit(project_object=project, run_config=run_config, script_to_run=train_script) print(helpers.get_run_history_url(run)) # Shows output of the run on stdout. run.wait_for_completion(show_output=True)
def main(): parser = argparse.ArgumentParser( description="NGC Set Up on AzureML Compute Cluster") parser.add_argument("--config_file", type=str, help="location of config file") args = parser.parse_args() config_file = args.config_file print(config_file) configdata = ngccontent.get_config(config_file) subscription_id = configdata["azureml_user"]["subscription_id"] resource_group = configdata["azureml_user"]["resource_group"] workspace_name = configdata["azureml_user"]["workspace_name"] ws = Workspace(workspace_name=workspace_name, subscription_id=subscription_id, resource_group=resource_group) verify = f''' Subscription ID: {subscription_id} Resource Group: {resource_group} Workspace: {workspace_name}''' print(verify) ### vnet settings vnet_rg = ws.resource_group vnet_name = configdata["aml_compute"]["vnet_name"] subnet_name = configdata["aml_compute"]["subnet_name"] ### azure ml names ct_name = configdata["aml_compute"]["ct_name"] exp_name = configdata["aml_compute"]["exp_name"] ### trust but verify verify = f''' vNET RG: {vnet_rg} vNET name: {vnet_name} vNET subnet name: {subnet_name} Compute target: {ct_name} Experiment name: {exp_name}''' print(verify) if configdata["aml_compute"]["vm_name"] in configdata[ "supported_vm_sizes"].keys(): vm_name = configdata["aml_compute"]["vm_name"] gpus_per_node = configdata["supported_vm_sizes"][vm_name] print( "Setting up compute target {ct_name} with vm_size: {vm_name} with {gpus_per_node} GPUs" .format(ct_name=ct_name, vm_name=vm_name, gpus_per_node=gpus_per_node)) if ct_name not in ws.compute_targets: config = AmlCompute.provisioning_configuration( vm_size=vm_name, min_nodes=configdata["aml_compute"]["min_nodes"], max_nodes=configdata["aml_compute"]["max_nodes"], vnet_resourcegroup_name=vnet_rg, vnet_name=vnet_name, subnet_name=subnet_name, idle_seconds_before_scaledown=configdata["aml_compute"] ["idle_seconds_before_scaledown"], remote_login_port_public_access='Enabled') ct = ComputeTarget.create(ws, ct_name, config) ct.wait_for_completion(show_output=True) else: print("Loading Pre-existing Compute Target {ct_name}".format( ct_name=ct_name)) ct = ws.compute_targets[ct_name] else: print("Unsupported vm_size {vm_size}".format(vm_size=vm_name)) print("The specified vm size must be one of ...") for azure_gpu_vm_size in configdata["supported_vm_sizes"].keys(): print("... " + azure_gpu_vm_size) raise Exception( "{vm_size} does not support Pascal or above GPUs".format( vm_size=vm_name)) environment_name = configdata["aml_compute"]["environment_name"] python_interpreter = configdata["aml_compute"]["python_interpreter"] conda_packages = configdata["aml_compute"]["conda_packages"] from azureml.core import ContainerRegistry if environment_name not in ws.environments: env = Environment(name=environment_name) env.docker.enabled = configdata["aml_compute"]["docker_enabled"] env.docker.base_image = None env.docker.base_dockerfile = "FROM {dockerfile}".format( dockerfile=configdata["ngc_content"]["base_dockerfile"]) env.python.interpreter_path = python_interpreter env.python.user_managed_dependencies = True conda_dep = CondaDependencies() for conda_package in conda_packages: conda_dep.add_conda_package(conda_package) env.python.conda_dependencies = conda_dep env.register(workspace=ws) evn = env else: env = ws.environments[environment_name] amlcluster = AzureMLComputeCluster( workspace=ws, compute_target=ct, initial_node_count=1, experiment_name=configdata["aml_compute"]["exp_name"], environment_definition=env, use_gpu=True, n_gpus_per_node=1, jupyter=True, jupyter_port=configdata["aml_compute"]["jupyter_port"], dashboard_port=9001, scheduler_port=9002, scheduler_idle_timeout=1200, worker_death_timeout=30, additional_ports=[], datastores=[], telemetry_opt_out=True, asynchronous=False) print(amlcluster.jupyter_link) amlcluster.jupyter_link print('Exiting script')
model = Model.register(model_path = filename, model_name = "textblob", tags = {"key": "2"}, description = "Sentiment Prediction", workspace = ws) aciconfig = AciWebservice.deploy_configuration(cpu_cores=1, memory_gb=1, tags={"data": "sentiment", "method" : "textblob"}, description='Predict Sentiment Score') textblobenv = CondaDependencies() textblobenv.add_conda_package("scikit-learn") # textblobenv.add_conda_package("textblob") # textblobenv.add_conda_package("pickle") # textblobenv.add_conda_package("dill") with open("textblobenv.yml","w") as f: f.write(textblobenv.serialize_to_string()) with open("textblobenv.yml","r") as f: print(f.read()) ############################# %%writefile score.py import json import numpy as np import os
}, description= "Output labels of the retrained Inception V3 model with flower photos", workspace=ws) print("Checking AKS state...") status = aks_target.get_status() while status != 'Succeeded' and status != 'Failed': print('current status: {} - waiting...'.format(status)) time.sleep(30) status = aks_target.get_status() print("Creating image and service configuration...") myenv = CondaDependencies() myenv.add_tensorflow_conda_package(core_type='cpu') myenv.add_conda_package("numpy") with open(os.path.join(project_folder, "myenv.yml"), "w") as f: f.write(myenv.serialize_to_string()) shutil.copy("./scripts/score_flowers.py", './') # configure the image image_config = ContainerImage.image_configuration( execution_script="score_flowers.py", runtime="python", conda_file=os.path.join(project_folder, "myenv.yml")) aks_config = AksWebservice.deploy_configuration( cpu_cores=1, memory_gb=1,
output_name='classification_data', is_directory=True) compute_target = ws.compute_targets['cpu-cluster'] environment_variables = { 'POSTGRES_PASSWORD': os.environ['POSTGRES_PASSWORD'], 'POSTGRES_HOSTNAME': 'ackbar-postgres.postgres.database.azure.com', 'AZURE_STORAGE_CONNECTION_STRING': os.environ['AZURE_STORAGE_CONNECTION_STRING'] } env = Environment(name='env', environment_variables=environment_variables) conda = CondaDependencies() conda.add_conda_package('psycopg2') conda.add_conda_package('numpy') conda.add_conda_package('Pillow') # have to use pip to install azure packages... conda.add_pip_package('azure-storage-blob') env.python.conda_dependencies = conda run_config = RunConfiguration() run_config.environment = env PROJECT = 'caltech' prepare_step = PythonScriptStep( script_name='prepare.py', arguments=['--output', batch_input, '--project', PROJECT], inputs=[], outputs=[batch_input],
x = F.dropout(x, training=self.training) x = x.view(-1, 32 * 32 * 24) x = self.fc(x) return F.log_softmax(x, dim=1) ''' Create an environment file The web service will be hosted in a container, and the container will need to install any Python dependencies when it gets initialized. In this case, our scoring code requires the torch and torchvision Python libraries, so we'll create a .yml file that tells the container host to install these into the environment along with the default libraries used by Azure ML. ''' from azureml.core.conda_dependencies import CondaDependencies myenv = CondaDependencies() myenv.add_conda_package("pytorch") myenv.add_conda_package("torchvision") myenv.add_channel("pytorch") env_file = "env_pytorch.yml" with open(env_file,"w") as f: f.write(myenv.serialize_to_string()) print("Saved dependency info in", env_file) with open(env_file,"r") as f: print(f.read()) ''' Deploy the web service
# + # Prepare environment config from azureml.core.conda_dependencies import CondaDependencies from azureml.core.model import InferenceConfig from azureml.core.environment import Environment if not 'temp' in os.listdir(): os.mkdir('temp') df_test.to_json("temp/test_sample.json") # save data for external tests # Create the environment myenv = Environment(name="mortgage_score_env_hd") conda_dep = CondaDependencies() # Define the packages needed by the model and scripts conda_dep.add_conda_package("numpy") conda_dep.add_conda_package("pip") conda_dep.add_conda_package("scikit-learn=0.20.3") # You must list azureml-defaults as a pip dependency conda_dep.add_pip_package("azureml-defaults==1.11.0") conda_dep.add_pip_package("azureml-core") conda_dep.add_pip_package("azureml-automl-runtime") conda_dep.add_pip_package("packaging") conda_dep.add_pip_package("azureml-explain-model==1.11.0") conda_dep.add_pip_package("inference-schema") conda_dep.add_conda_package("numpy") # scikit-learn>=0.19.0,<=0.20.3 conda_dep.add_conda_package("pandas") conda_dep.add_conda_package("py-xgboost") # Save environment also locally to disk so we can test the score script directly by creating a local environment conda_dep.save('temp/mortgage_score_env.yml')
model = Model.register(model_name=MODEL_NAME, model_path=MODEL_PATH, workspace=ws, description=MODEL_DESCRIPTION) print(model.name, model.id, model.version, sep='\t') #Set the image aciconfig = AciWebservice.deploy_configuration(cpu_cores=CPU_CORES, memory_gb=MEMORY_GB, description=SERVICE_DESCRIPTION, auth_enabled=AUTH_ENABLED) if CONDA_FILE_URL == '' and DOCKER_FILE_URL == '': from azureml.core.conda_dependencies import CondaDependencies myenv = CondaDependencies() myenv.add_conda_package("scikit-learn") #myenv.add_pip_package("joblib") with open("myenv.yml", "w") as f: f.write(myenv.serialize_to_string()) # configure the image image_config = ContainerImage.image_configuration( execution_script=EXECUTION_SCRIPT_PATH, runtime="python", conda_file="myenv.yml") elif CONDA_FILE_URL is not '' and DOCKER_FILE_URL == '': wget.download(CONDA_FILE_URL, CONDA_FILE_PATH) image_config = ContainerImage.image_configuration( execution_script=EXECUTION_SCRIPT_PATH, runtime="python", conda_file=CONDA_FILE_PATH) elif DOCKER_FILE_URL is not '':
# register model from azureml.core import Run model = Model.register(ws, model_name='objectmodelchal5', model_path='objectmodelchal5.h5') print(model.name, model.id, model.version, sep = '\t') # ## YAML EXPORT # In[90]: from azureml.core.conda_dependencies import CondaDependencies myenv = CondaDependencies() myenv.add_conda_package("keras") myenv.add_conda_package("pillow") myenv.add_conda_package("numpy") myenv.add_conda_package("requests") myenv.add_conda_package("h5py") with open("myenv.yml","w") as f: f.write(myenv.serialize_to_string()) # In[91]: model=Model(ws, 'objectmodelchal5') model.download(target_dir = '.')
provisioning_config) compute_target.wait_for_provisioning(show_output=True) print('create Batch AI run config') rc = RunConfiguration(project, "dask_run_config") rc.environment.docker.enabled = True rc.prepare_environment = True rc.target = batchai_cluster_name rc.environment.python.user_managed_dependencies = False rc.batchai.node_count = 2 # create a new CondaDependencies obj cd = CondaDependencies() # add scikit-learn as a conda dependency cd.add_conda_package('dask') cd.add_conda_package('joblib') cd.add_pip_package('azureml-contrib-daskonbatch') # overwrite the default conda_dependencies.yml file cd.save_to_file(project_dir=project_folder, file_name='conda_dependencies.yml') print() print('##################################################') print('submitting {} for a batch ai run...'.format(train_script)) print('##################################################') print() print("prepare run...") prep = Run.prepare_compute_target(project_object=project, run_config=rc)
def conda_deps(): deps = CondaDependencies(f'{project_folder}/environment.yml') deps.add_channel("conda-forge") deps.add_conda_package('curl') return deps
from azureml.core.model import InferenceConfig from azureml.core.environment import Environment from azureml.core.conda_dependencies import CondaDependencies # Create the environment myenv = Environment(name="myenv") conda_dep = CondaDependencies() # Define the packages needed by the model and scripts conda_dep.add_conda_package("tensorflow") conda_dep.add_conda_package("numpy") conda_dep.add_conda_package("scikit-learn") # You must list azureml-defaults as a pip dependency conda_dep.add_pip_package("azureml-defaults") conda_dep.add_pip_package("keras") conda_dep.add_pip_package("gensim") # Adds dependencies to PythonSection of myenv myenv.python.conda_dependencies = conda_dep inference_config = InferenceConfig(entry_script="score.py", environment=myenv)
ws = load_workspace() # Grab the model object from the list of available models model_list = Model.list(workspace=ws) model = None # Unpack the generator and look through the list to find your desired model model, = (m for m in model_list if m.version == model_version and m.name == model_name) print( 'Model picked: {} \nModel Description: {} \nModel Version: {}'.format( model.name, model.description, model.version)) dependencies = CondaDependencies() dependencies.add_conda_package("numpy") dependencies.add_conda_package("matplotlib") dependencies.add_conda_package("scikit-learn") dependencies.add_conda_package("tensorflow") dependencies.add_conda_package("keras") dependencies.add_conda_package("scikit-image") dependencies.add_pip_package("pynacl==1.2.1") os.makedirs("./score/", exist_ok=True) with open("./score/dependencies.yml", "w") as f: f.write(dependencies.serialize_to_string()) original_dir = os.getcwd() # Change directory since the docker container is expecting thing at the TLD os.chdir("./score") image_config = ContainerImage.image_configuration(
model_path = Model.get_model_path("classi_model") model = joblib.load(model_path) def run(raw_data): data = np.array(json.loads(raw_data)['data']) pred = model.predict(data) return pred.tolist() # Need to create a script for Environment from azureml.core.conda_dependencies import CondaDependencies my_env = CondaDependencies() my_env.add_conda_package('scikit-learn') env_file = 'service_files/env.yml' with open(env_file, "w") as f: f.write(my_env.serialize_to_string()) print("Saved dependency info in", env_file) #combining both of the script in inference config from azureml.core.model import InferenceConfig class_inference_config = InferenceConfig(runtime='python', source_directory='service_files', entry_script='score.py',
from azureml.core.model import Model from azureml.core import Workspace ws = Workspace.from_config(path=".azureml/config.json") # Inference Configuration from azureml.core.model import InferenceConfig from azureml.core.environment import Environment from azureml.core.conda_dependencies import CondaDependencies # Create the environment myenv = Environment(name="myenv") conda_dep = CondaDependencies() # Define the packages needed by the model and scripts conda_dep.add_conda_package("python=3.6.2") conda_dep.add_conda_package("numpy=1.18.5") conda_dep.add_conda_package("scikit-learn") conda_dep.add_conda_package("pip") # You must list azureml-defaults as a pip dependency conda_dep.add_pip_package("azureml-defaults") conda_dep.add_pip_package("tensorflow==2.3.0") conda_dep.add_pip_package("keras==2.4.3") # Adds dependencies to PythonSection of myenv myenv.python.conda_dependencies = conda_dep inference_config = InferenceConfig(entry_script="./source_dir/score.py", environment=myenv) # Deployment config
run.log('averge: ', average) run.log("Experiment end time", str(datetime.datetime.now())) run.complete() model = Model.register(model_path = filename, model_name = "ta_model", tags = {"key": "1"}, description = "TextBlob Prediction", workspace = ws) ta_env = CondaDependencies() ta_env.add_conda_package("scikit-learn") ta_env.add_conda_package("dill") with open("ta_env.yml","w") as f: f.write(ta_env.serialize_to_string()) with open("ta_env.yml","r") as f: print(f.read()) %%time image_config = ContainerImage.image_configuration(execution_script="score.py", runtime="python", conda_file="ta_env.yml")
df = pd.DataFrame([data[1:]],columns=featurenames) # make prediction if data[0] == 'iq': result = iq_model.predict(df).astype(int) elif data[0] == 'sj': result = sj_model.predict(df).astype(int) # you can return any data type as long as it is JSON-serializable return result.tolist() #%% create environment file for deployment from azureml.core.conda_dependencies import CondaDependencies mymodelenv = CondaDependencies() mymodelenv.add_conda_package("scikit-learn") mymodelenv.add_conda_package("pandas") mymodelenv.add_conda_package("statsmodels") mymodelenv.add_conda_package("scipy=1.2") mymodelenv.add_conda_package("numpy") with open("mymodelenv.yml","w") as f: f.write(mymodelenv.serialize_to_string()) with open("mymodelenv.yml","r") as f: print(f.read()) print('Complete') #%% create container image
run_config.container_instance.memory_gb = 2 # enable Docker run_config.environment.docker.enabled = True # set Docker base image to the default CPU-based image run_config.environment.docker.base_image = azureml.core.runconfig.DEFAULT_MMLSPARK_CPU_IMAGE print('base image is', run_config.environment.docker.base_image) #run_config.environment.docker.base_image = 'microsoft/mmlspark:plus-0.9.9' # use conda_dependencies.yml to create a conda environment in the Docker image for execution # please update this file if you need additional packages. run_config.environment.python.user_managed_dependencies = False cd = CondaDependencies() cd.add_conda_package('numpy') # overwrite the default conda_dependencies.yml file cd.save_to_file(project_dir = project_folder, file_name='conda_dependencies.yml') # auto-prepare the Docker image when used for execution (if it is not already prepared) run_config.prepare_environment = True print() print('##################################################') print('submitting {} for a Spark run on ACI...'.format(train_script)) print('##################################################') print() run = Run.submit(project_object = project, run_config = run_config, script_to_run = "train-spark.py")
def generate_yaml( directory: str, ref_filename: str, needed_libraries: list, conda_filename: str, ): """ Creates a deployment-specific yaml file as a subset of the image classification environment.yml Also adds extra libraries, if not present in environment.yml Args: directory (string): Directory name of reference yaml file ref_filename (string): Name of reference yaml file needed_libraries (list of strings): List of libraries needed in the Docker container conda_filename (string): Name of yaml file to be deployed in the Docker container Returns: Nothing """ with open(os.path.join(directory, ref_filename), "r") as f: yaml_content = yaml.load(f, Loader=yaml.FullLoader) # Extract libraries to be installed using conda extracted_libraries = [ depend for depend in yaml_content["dependencies"] if any(lib in depend for lib in needed_libraries) ] # Extract libraries to be installed using pip if any(isinstance(x, dict) for x in yaml_content["dependencies"]): # if the reference yaml file contains a "pip" section, # find where it is in the list of dependencies ind = [ yaml_content["dependencies"].index(depend) for depend in yaml_content["dependencies"] if isinstance(depend, dict) ][0] extracted_libraries += [ depend for depend in yaml_content["dependencies"][ind]["pip"] if any(lib in depend for lib in needed_libraries) ] # Check whether additional libraries are needed not_found = [ lib for lib in needed_libraries if not any(lib in ext for ext in extracted_libraries) ] # Create the deployment-specific yaml file conda_env = CondaDependencies() for ch in yaml_content["channels"]: conda_env.add_channel(ch) for library in extracted_libraries + not_found: conda_env.add_conda_package(library) # Display the environment print(conda_env.serialize_to_string()) # Save the file to disk conda_env.save_to_file(base_directory=os.getcwd(), conda_file_path=conda_filename)
data_reference_name="input_data", path_on_datastore="churn") processed_dir = PipelineData(name='processed_data', datastore=default_store) #%% [markdown] # ## Pipeline 1st step: Data Preprocessing # # We start by defining the run configuration with the needed dependencies by the preprocessing step. # # In the cell that follow, we compose the first step of the pipeline. # #%% cd = CondaDependencies() cd.add_conda_package('pandas') cd.add_conda_package('matplotlib') cd.add_conda_package('numpy') cd.add_conda_package('scikit-learn') run_config = RunConfiguration(framework="python", conda_dependencies=cd) run_config.target = cluster run_config.environment.docker.enabled = True run_config.environment.docker.base_image = DEFAULT_GPU_IMAGE run_config.environment.python.user_managed_dependencies = False #%% pre_processing = PythonScriptStep( name='preprocess dataset', script_name='preprocess.py', arguments=['--input_path', input_dir,\
import azureml.core from azureml.core import Workspace from azureml.core.authentication import InteractiveLoginAuthentication from azureml.core.conda_dependencies import CondaDependencies from azureml.core.model import Model from azureml.core.image import ContainerImage from azureml.core.webservice import AciWebservice from azureml.core.webservice import Webservice auth_config = InteractiveLoginAuthentication(False, "72f988bf-86f1-41af-91ab-2d7cd011db47") ws=Workspace.from_config('aml_config/config.json', auth_config) ws.get_details() myenv = CondaDependencies() myenv.add_conda_package("keras") myenv.add_conda_package("tensorflow") myenv.add_conda_package("pillow") with open("myenv.yml","w") as f: f.write(myenv.serialize_to_string()) # Register a trained model print('Registering model...') model = Model.register(model_path = "modelfiles", model_name = "dogs-vs-cat", description = "ready lab 314", workspace = ws) # Image configuration print('Creating image configuration...') image_config = ContainerImage.image_configuration(execution_script = "score.py",