sourceDirectory = "./code" remoteComputeTargetName = "default" # make this a parameter computeTarget = ws.compute_targets[remoteComputeTargetName] run_config = RunConfiguration(conda_dependencies=CondaDependencies.create( conda_packages=['numpy', 'pandas', 'scikit-learn', 'tensorflow', 'keras'], pip_packages=[ 'azure', 'azureml-core', 'azure-storage', 'azure-storage-blob', 'azureml-dataprep' ])) run_config.environment.docker.enabled = True ##------------- Pipeline Parameters datasetName = PipelineParameter(name="datasetName", default_value="qualitydataset") datasetStorePath = PipelineParameter(name="datasetStorePath", default_value="/inputdata/train.csv") modelName = PipelineParameter(name="modelName", default_value="quality_gbm_model.pkl") ##------------- Data preprocessing step preprocessingStep = PythonScriptStep(name="preprocess_step", script_name="preprocess.py", source_directory=sourceDirectory, compute_target=computeTarget, arguments=[ "--datasetName", datasetName, "--datasetStorePath", datasetStorePath,
#copy all necessary scripts files = FilesProviders.get_path_files( "../", [os.path.basename(__file__), "__init__.py"]) for f in files: shutil.copy(f, script_folder) #add generated config file to script folder shutil.copy(generated_config_file, script_folder) input_data = input_dataset.as_named_input('input_dataset').as_mount() data_store = ws.get_default_datastore() prepped_data = PipelineData('prepped_data', datastore=data_store) pipeline_mode_param = PipelineParameter(name="mode", default_value="execute") prep_step = PythonScriptStep( name='Prepare data', source_directory=script_folder, script_name='prep_data.py', compute_target=compute_target, runconfig=pipeline_run_config, # Specify dataset as initial input inputs=[input_data], # Specify PipelineData as output outputs=[prepped_data], # Also pass as data reference to script arguments=[ '--input_data', input_data, '--prepped_data', prepped_data, '--mode', pipeline_mode_param
def main(): e = ENV() aml_workspace = Workspace.get( name=e.workspace_name, subscription_id=e.subscription_id, resource_group=e.resource_group ) print(f'workspace:{aml_workspace}') aml_compute = get_compute(workspace=aml_workspace, compute_name=e.compute_name, vm_size=e.vm_size) if aml_compute is not None: print(f'compute target: {aml_compute} is created') environment = get_environment(workspace=aml_workspace, env_name=e.aml_env_name, conda_dependencies=e.aml_conda_train_dependent_files, create_new=e.rebuild_env) run_config = RunConfiguration() run_config.environment = environment if e.datastore_name: datastore_name = e.datastore_name else: datastore_name = aml_workspace.get_default_datastore().name run_config.environment.environment_variables['DATASTORE_NAME'] = datastore_name model_name_param = PipelineParameter(name='model_name',default_value=e.model_name) dataset_version_param = PipelineParameter(name='dataset_version',default_value=e.dataset_version) dataset_file_path = PipelineParameter(name='dataset_file_path',default_value='none') dataset_name = e.dataset_name if dataset_name not in aml_workspace.datasets: create_sample_data_csv() file_name = 'diabetes.csv' if not os.path.exists(file_name): raise Exception(f'{file_name} does not exist!') datastore = Datastore.get(aml_workspace,datastore_name) target_path = 'training/' datastore.upload_files(files=[file_name], target_path=target_path, overwrite=True, show_progress=True) path_on_datastore = os.path.join(target_path,file_name) dataset = Dataset.Tabular.from_delimited_files(path=(datastore,path_on_datastore)) dataset.register(workspace=aml_workspace, name=dataset_name, description='registered dataset', create_new_version=True, tags={'format':'CSV'}) pipeline_data = PipelineData('train',datastore=aml_workspace.get_default_datastore()) train_step = PythonScriptStep(script_name=e.train_script_path, name='train_step', arguments=['--model-name',model_name_param, '--dataset-name',dataset_name, '--dataset-version',dataset_version_param, '--dataset-file-path',dataset_file_path, '--step-output',pipeline_data], compute_target=aml_compute, runconfig=run_config, source_directory=e.source_train_directory, outputs=[pipeline_data], allow_reuse=False) print('Train step created!') eval_step = PythonScriptStep(name='eval_step', script_name=e.eval_script_path, compute_target=aml_compute, source_directory=e.source_train_directory, arguments=['--model-name',model_name_param, '--allow-run-cancel',e.allow_run_cancel], runconfig=run_config, allow_reuse=False) print('EVAL step created!') register_step = PythonScriptStep(name='register_step', script_name=e.register_script_path, compute_target=aml_compute, source_directory=e.source_train_directory, inputs=[pipeline_data], arguments=['--model-name',model_name_param, '--step-input',pipeline_data], runconfig=run_config, allow_reuse=False) print('Register step created!') if e.run_evaluation: print('evaluation step is included') eval_step.run_after(train_step) register_step.run_after(eval_step) steps = [train_step,eval_step,register_step] else: print('evaluation step is excluded') register_step.run_after(train_step) steps = [train_step,register_step] train_pipeline = Pipeline(workspace=aml_workspace, steps=steps) train_pipeline.validate() published_pipeline = train_pipeline.publish(name=e.pipeline_name, description='model training pipeline', version=e.build_id) print(f'{published_pipeline.name} is published1')
def main(): load_dotenv() workspace_name = os.environ.get("BASE_NAME") + "-AML-WS" resource_group = "AML-RG-" + os.environ.get("BASE_NAME") subscription_id = os.environ.get("SUBSCRIPTION_ID") tenant_id = os.environ.get("TENANT_ID") app_id = os.environ.get("SP_APP_ID") app_secret = os.environ.get("SP_APP_SECRET") sources_directory_train = os.environ.get("SOURCES_DIR_TRAIN") train_script_path = os.environ.get("TRAIN_SCRIPT_PATH") evaluate_script_path = os.environ.get("EVALUATE_SCRIPT_PATH") vm_size = os.environ.get("AML_COMPUTE_CLUSTER_SKU") compute_name = os.environ.get("AML_COMPUTE_CLUSTER_NAME") aks_name = os.environ.get("AKS_CLUSTER_NAME") model_name = os.environ.get("MODEL_NAME") build_id = os.environ.get("BUILD_BUILDID") pipeline_name = os.environ.get("TRAINING_PIPELINE_NAME") experiment_name = os.environ.get("EXPERIMENT_NAME") # Get Azure machine learning workspace aml_workspace = get_workspace(workspace_name, resource_group, subscription_id, tenant_id, app_id, app_secret) print(aml_workspace) # Get Azure machine learning cluster aml_compute = get_compute(aml_workspace, compute_name, vm_size) if aml_compute is not None: print(aml_compute) run_config = RunConfiguration(conda_dependencies=CondaDependencies.create( conda_packages=['numpy', 'pandas', 'scikit-learn', 'keras'], pip_packages=[ 'azure', 'azureml-sdk', 'azure-storage', 'azure-storage-blob', 'transformers>=2.1.1', 'tensorflow>=2.0.0', 'tensorflow-gpu>=2.0.0' ])) run_config.environment.docker.enabled = True datastore_name = 'tfworld' container_name = 'azure-service-classifier' account_name = 'johndatasets' sas_token = '?sv=2019-02-02&ss=bfqt&srt=sco&sp=rl&se=2021-06-02T03:40:25Z&st=2020-03-09T19:40:25Z&spr=https&sig=bUwK7AJUj2c%2Fr90Qf8O1sojF0w6wRFgL2c9zMVCWNPA%3D' try: existing_datastore = Datastore.get(aml_workspace, datastore_name) except: # noqa: E722 existing_datastore = Datastore \ .register_azure_blob_container(workspace=aml_workspace, datastore_name=datastore_name, container_name=container_name, account_name=account_name, sas_token=sas_token ) azure_dataset = Dataset.File.from_files(path=(existing_datastore, 'data')) azure_dataset = azure_dataset.register( workspace=aml_workspace, name='Azure Services Dataset', description='Dataset containing azure related posts on Stackoverflow', create_new_version=True) azure_dataset.to_path() input_data = azure_dataset.as_named_input('input_data1').as_mount( '/tmp/data') model_name = PipelineParameter(name="model_name", default_value=model_name) max_seq_length = PipelineParameter(name="max_seq_length", default_value=128) learning_rate = PipelineParameter(name="learning_rate", default_value=3e-5) num_epochs = PipelineParameter(name="num_epochs", default_value=3) export_dir = PipelineParameter(name="export_dir", default_value="./outputs/exports") batch_size = PipelineParameter(name="batch_size", default_value=32) steps_per_epoch = PipelineParameter(name="steps_per_epoch", default_value=100) # initialize the TensorFlow estimator estimator = TensorFlow(source_directory=sources_directory_train, entry_script=train_script_path, compute_target=aml_compute, framework_version='2.0', use_gpu=True, pip_packages=[ 'transformers==2.0.0', 'azureml-dataprep[fuse,pandas]==1.3.0' ]) train_step = EstimatorStep( name="Train Model", estimator=estimator, estimator_entry_script_arguments=[ "--data_dir", input_data, "--max_seq_length", max_seq_length, "--learning_rate", learning_rate, "--num_epochs", num_epochs, "--export_dir", export_dir, "--batch_size", batch_size, "--steps_per_epoch", steps_per_epoch ], compute_target=aml_compute, inputs=[input_data], allow_reuse=False, ) print("Step Train created") evaluate_step = PythonScriptStep( name="Evaluate Model ", script_name=evaluate_script_path, compute_target=aml_compute, source_directory=sources_directory_train, arguments=[ "--model_name", model_name, "--build_id", build_id, ], runconfig=run_config, allow_reuse=False, ) print("Step Evaluate created") # Currently, the Evaluate step will automatically register # the model if it performs better. This step is based on a # previous version of the repo which utilized JSON files to # track evaluation results. evaluate_step.run_after(train_step) steps = [evaluate_step] train_pipeline = Pipeline(workspace=aml_workspace, steps=steps) train_pipeline.validate() published_pipeline = train_pipeline.publish( name=pipeline_name, description="Model training/retraining pipeline", version=build_id) print(f'Published pipeline: {published_pipeline.name}') print(f'for build {published_pipeline.version}') response = published_pipeline.submit( # noqa: F841 workspace=aml_workspace, experiment_name=experiment_name) # Get AKS cluster for deployment aks_compute = get_aks(aml_workspace, aks_name) if aks_compute is not None: print(aks_compute)
def main(): load_dotenv() workspace_name = os.environ.get("BASE_NAME") + "-AML-WS" resource_group = os.environ.get("BASE_NAME") + "-AML-RG" subscription_id = os.environ.get("SUBSCRIPTION_ID") tenant_id = os.environ.get("TENANT_ID") app_id = os.environ.get("SP_APP_ID") app_secret = os.environ.get("SP_APP_SECRET") sources_directory_train = os.environ.get("SOURCES_DIR_TRAIN") train_script_path = os.environ.get("TRAIN_SCRIPT_PATH") evaluate_script_path = os.environ.get("EVALUATE_SCRIPT_PATH") # register_script_path = os.environ.get("REGISTER_SCRIPT_PATH") vm_size = os.environ.get("AML_COMPUTE_CLUSTER_CPU_SKU") compute_name = os.environ.get("AML_COMPUTE_CLUSTER_NAME") model_name = os.environ.get("MODEL_NAME") build_id = os.environ.get("BUILD_BUILDID") pipeline_name = os.environ.get("TRAINING_PIPELINE_NAME") print(app_secret) # Get Azure machine learning workspace aml_workspace = get_workspace(workspace_name, resource_group, subscription_id, tenant_id, app_id, app_secret) print(aml_workspace) # Get Azure machine learning cluster aml_compute = get_compute(aml_workspace, compute_name, vm_size) if aml_compute is not None: print(aml_compute) run_config = RunConfiguration(conda_dependencies=CondaDependencies.create( conda_packages=[ 'numpy', 'pandas', 'scikit-learn', 'tensorflow', 'keras' ], pip_packages=[ 'azure', 'azureml-core', 'azure-storage', 'azure-storage-blob' ])) run_config.environment.docker.enabled = True model_name = PipelineParameter(name="model_name", default_value=model_name) release_id = PipelineParameter(name="release_id", default_value="0") train_step = PythonScriptStep( name="Train Model", script_name=train_script_path, compute_target=aml_compute, source_directory=sources_directory_train, arguments=[ "--release_id", release_id, "--model_name", model_name, ], runconfig=run_config, allow_reuse=False, ) print("Step Train created") evaluate_step = PythonScriptStep( name="Evaluate Model ", script_name=evaluate_script_path, compute_target=aml_compute, source_directory=sources_directory_train, arguments=[ "--release_id", release_id, "--model_name", model_name, ], runconfig=run_config, allow_reuse=False, ) print("Step Evaluate created") # Currently, the Evaluate step will automatically register # the model if it performs better. This step is based on a # previous version of the repo which utilized JSON files to # track evaluation results. # register_model_step = PythonScriptStep( # name="Register New Trained Model", # script_name=register_script_path, # compute_target=aml_compute, # source_directory=sources_directory_train, # arguments=[ # "--release_id", release_id, # "--model_name", model_name, # ], # runconfig=run_config, # allow_reuse=False, # ) # print("Step register model created") evaluate_step.run_after(train_step) # register_model_step.run_after(evaluate_step) steps = [evaluate_step] train_pipeline = Pipeline(workspace=aml_workspace, steps=steps) train_pipeline.validate() published_pipeline = train_pipeline.publish( name=pipeline_name, description="Model training/retraining pipeline", version=build_id) print(f'Published pipeline: {published_pipeline.name}') print(f'for build {published_pipeline.version}')
def main(): e = Env() # Get Azure machine learning workspace aml_workspace = Workspace.get( name=e.workspace_name, subscription_id=e.subscription_id, resource_group=e.resource_group, ) print(f"get_workspace:{aml_workspace}") # Get Azure machine learning cluster aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size) if aml_compute is not None: print(f"aml_compute:{aml_compute}") # Create a reusable Azure ML environment environment = get_environment( aml_workspace, e.aml_env_name, create_new=e.rebuild_env, enable_docker=True, dockerfile='ml_model/preprocess/Dockerfile' ) # run_config = RunConfiguration() run_config.environment = environment if e.datastore_name: datastore_name = e.datastore_name else: datastore_name = aml_workspace.get_default_datastore().name run_config.environment.environment_variables["DATASTORE_NAME"] = datastore_name # NOQA: E501 datastore = Datastore(aml_workspace, name=datastore_name) data_file_path_param = PipelineParameter(name="data_file_path", default_value=e.dataset_name) # NOQA: E501 # The version of the input/output dataset can't be determined at pipeline publish time, only run time. # NOQA: E501 # Options to store output data: # Option 1: Use blob API to write output data. Otherwise, no way to dynamically change the output dataset based on PipelineParameter, # NOQA: E501 # The following will not work. It generate a path like "PipelineParameter_Name:data_file_path_Default:gear_images" # NOQA: E501 # output_ds = OutputFileDatasetConfig(destination=(datastore, data_file_path_param)) # NOQA: E501 # This option means writing a file locally and upload to the datastore. Fewer dataset, more code. # NOQA: E501 # Option 2: Use a dynamic path in OutputFileDatasetConfig, and register a new dataset at completion # NOQA: E501 # Output dataset can be mounted, so more dataset to maintain, less code. # NOQA: E501 # Using Option 2 below. output_dataset = OutputFileDatasetConfig( name=e.processed_dataset_name, destination=(datastore, "/dataset/{output-name}/{run-id}") ).register_on_complete( name=e.processed_dataset_name) preprocess_step = PythonScriptStep( name="Preprocess Data with OS cmd", script_name='preprocess/preprocess_os_cmd_aml.py', compute_target=aml_compute, source_directory=e.sources_directory_train, arguments=[ "--dataset_name", e.dataset_name, "--datastore_name", datastore_name, "--data_file_path", data_file_path_param, "--output_dataset", output_dataset, ], runconfig=run_config, allow_reuse=False, ) print("Step Preprocess OS cmd created") steps = [preprocess_step] preprocess_pipeline = Pipeline(workspace=aml_workspace, steps=steps) preprocess_pipeline._set_experiment_name preprocess_pipeline.validate() published_pipeline = preprocess_pipeline.publish( name=e.preprocessing_pipeline_name, description="Data preprocessing OS cmd pipeline", version=e.build_id, ) print(f"Published pipeline: {published_pipeline.name}") print(f"for build {published_pipeline.version}")
def main(): load_dotenv() workspace_name = os.environ.get("WS_NAME") resource_group = os.environ.get("RG_NAME") subscription_id = os.environ.get("SUBSCRIPTION_ID") tenant_id = os.environ.get("TENANT_ID") app_id = os.environ.get("SP_APP_ID") app_secret = os.environ.get("SP_APP_SECRET") sources_directory_train = os.environ.get("SOURCES_DIR_TRAIN") train_script_path = os.environ.get("TRAIN_SCRIPT_PATH") evaluate_script_path = os.environ.get("EVALUATE_SCRIPT_PATH") vm_size = os.environ.get("AML_COMPUTE_CLUSTER_SKU") compute_name = os.environ.get("AML_COMPUTE_CLUSTER_NAME") aks_name = os.environ.get("AKS_CLUSTER_NAME") model_name = os.environ.get("MODEL_NAME") build_id = os.environ.get("BUILD_BUILDID") pipeline_name = os.environ.get("TRAINING_PIPELINE_NAME") experiment_name = os.environ.get("EXPERIMENT_NAME") # Get Azure machine learning workspace aml_workspace = get_workspace( workspace_name, resource_group, subscription_id, tenant_id, app_id, app_secret) print('Now accessing:') print(aml_workspace) # Get Azure machine learning cluster aml_compute = get_compute( aml_workspace, compute_name, vm_size) if aml_compute is not None: print(aml_compute) run_config = RunConfiguration(conda_dependencies=CondaDependencies.create( conda_packages=['numpy', 'pandas', 'scikit-learn', 'keras'], pip_packages=['azureml-core==1.25.0', 'azureml-defaults==1.25.0', 'azureml-telemetry==1.25.0', 'azureml-train-restclients-hyperdrive==1.25.0', 'azureml-train-core==1.25.0', 'azureml-dataprep', 'tensorflow-gpu==2.0.0', 'transformers==2.0.0', 'absl-py', 'azureml-dataprep', 'h5py<3.0.0']) ) # run_config.environment.docker.enabled = True datastore_name = 'mtcseattle' container_name = 'azure-service-classifier' account_name = 'mtcseattle' sas_token = '?sv=2020-04-08&st=2021-05-26T04%3A39%3A46Z&se=2022-05-27T04%3A39%3A00Z&sr=c&sp=rl&sig=CTFMEu24bo2X06G%2B%2F2aKiiPZBzvlWHELe15rNFqULUk%3D' try: existing_datastore = Datastore.get(aml_workspace, datastore_name) except: # noqa: E722 existing_datastore = Datastore \ .register_azure_blob_container(workspace=aml_workspace, datastore_name=datastore_name, container_name=container_name, account_name=account_name, sas_token=sas_token, overwrite=True) azure_dataset = Dataset.File.from_files( path=(existing_datastore, 'data')) azure_dataset = azure_dataset.register( workspace=aml_workspace, name='Azure Services Dataset', description='Dataset containing azure related posts on Stackoverflow', create_new_version=True) azure_dataset.to_path() input_data = azure_dataset.as_named_input('azureservicedata').as_mount( '/tmp/data') model_name = PipelineParameter( name="model_name", default_value=model_name) max_seq_length = PipelineParameter( name="max_seq_length", default_value=128) learning_rate = PipelineParameter( name="learning_rate", default_value=3e-5) num_epochs = PipelineParameter( name="num_epochs", default_value=1) export_dir = PipelineParameter( name="export_dir", default_value="./outputs/model") batch_size = PipelineParameter( name="batch_size", default_value=32) steps_per_epoch = PipelineParameter( name="steps_per_epoch", default_value=1) # initialize the PythonScriptStep train_step = PythonScriptStep( name='Train Model', script_name=train_script_path, arguments=['--data_dir', input_data, '--max_seq_length', max_seq_length, '--batch_size', batch_size, '--learning_rate', learning_rate, '--steps_per_epoch', steps_per_epoch, '--num_epochs', num_epochs, '--export_dir',export_dir], compute_target=aml_compute, source_directory=sources_directory_train, runconfig=run_config, allow_reuse=True) print("Step Train created") evaluate_step = PythonScriptStep( name="Evaluate Model ", script_name=evaluate_script_path, compute_target=aml_compute, source_directory=sources_directory_train, arguments=[ "--model_name", model_name, "--build_id", build_id, ], runconfig=run_config, allow_reuse=False, ) print("Step Evaluate created") # Currently, the Evaluate step will automatically register # the model if it performs better. This step is based on a # previous version of the repo which utilized JSON files to # track evaluation results. evaluate_step.run_after(train_step) steps = [evaluate_step] train_pipeline = Pipeline(workspace=aml_workspace, steps=steps) train_pipeline.validate() published_pipeline = train_pipeline.publish( name=pipeline_name, description="Model training/retraining pipeline.", version=build_id ) print(f'Published pipeline: {published_pipeline.name}') print(f'for build {published_pipeline.version}') response = published_pipeline.submit( # noqa: F841 workspace=aml_workspace, experiment_name=experiment_name)
def main(): env = EnvironmentVariables() args = add_arguments() workspace = get_workspace() cpu_cluster_name = env.cpu_cluster_name compute = get_or_create_compute(workspace, cpu_cluster_name, env.compute_vm_size, env.max_nodes) environment = Environment.load_from_directory(env.sources_directory_train) environment.register(workspace) run_configuration = RunConfiguration() run_configuration.environment = environment model_name_param = PipelineParameter(name="model_name", default_value=env.model_name) build_id_param = PipelineParameter(name="build_id", default_value=env.build_id) should_tune_hyperparameters_param = PipelineParameter( name="should_tune_hyperparameters", default_value=env.should_tune_hyperparameters) parallelism_level_param = PipelineParameter( name="parallelism_level", default_value=env.parallelism_level) force_register_param = PipelineParameter(name="force_register", default_value=env.force_register) datastore = get_datastore() dataset_name = env.dataset_name dataset_path = env.dataset_path print( f"Creating new dataset version for {dataset_name} in datastore {datastore} from file {dataset_path}" ) temp_dataset = Dataset.Tabular.from_delimited_files(path=[(datastore, dataset_path)]) dataset = temp_dataset.register(workspace=workspace, name=dataset_name, description=dataset_name, tags={'format': 'CSV'}, create_new_version=True) train_output = PipelineData('train_output', output_name='train_output', datastore=datastore) train_step = PythonScriptStep( name="Train model", compute_target=compute, script_name=env.train_script_name, runconfig=run_configuration, inputs=[dataset.as_named_input('training')], outputs=[train_output], arguments=[ "--build_id", build_id_param, "--model_name", model_name_param, "--parallelism_level", parallelism_level_param, "--should_tune_hyperparameters", should_tune_hyperparameters_param ], allow_reuse=False) evaluate_step = PythonScriptStep(name="Evaluate model", compute_target=compute, script_name=env.evaluate_script_name, runconfig=run_configuration, inputs=[train_output], arguments=[ "--build_id", build_id_param, "--model_name", model_name_param, "--train_output", train_output, "--force_register", force_register_param ], allow_reuse=False) register_step = PythonScriptStep(name="Register model", compute_target=compute, script_name=env.register_script_name, runconfig=run_configuration, inputs=[train_output], arguments=[ "--build_id", build_id_param, "--model_name", model_name_param, "--train_output", train_output ], allow_reuse=False) evaluate_step.run_after(train_step) register_step.run_after(evaluate_step) steps = [train_step, evaluate_step, register_step] train_pipeline = Pipeline(workspace=workspace, steps=steps) train_pipeline.validate() published_pipeline = train_pipeline.publish( name=env.pipeline_name, description="Train/Eval/Register if better pipeline", version=env.build_id) output_file_name = args.output_file_name if output_file_name: with open(output_file_name, "w") as output_file: output_file.write(published_pipeline.id) print( f"Published pipeline {published_pipeline.name} for build {published_pipeline.version}" )
def main(): load_dotenv() workspace_name = os.environ.get("WORKSPACE_NAME") resource_group = os.environ.get("RESOURCE_GROUP_NAME") subscription_id = os.environ.get("SUBSCRIPTION_ID") tenant_id = os.environ.get("TENANT_ID") app_id = os.environ.get("SP_APP_ID") app_secret = os.environ.get("SP_APP_SECRET") sources_directory_train = os.environ.get("SOURCES_DIR_TRAIN") train_script_path = os.environ.get("TRAIN_SCRIPT_PATH") evaluate_script_path = os.environ.get("EVALUATE_SCRIPT_PATH") generate_report_path = os.environ.get("GENERATE_REPORT_PATH") generate_report_name = os.environ.get("GENERATE_REPORT_NAME") vm_size = os.environ.get("AML_COMPUTE_CLUSTER_GPU_SKU") compute_name = os.environ.get("AML_COMPUTE_CLUSTER_NAME") model_name = os.environ.get("MODEL_NAME") ckpt_path = os.environ.get("MODEL_CHECKPOINT_PATH") build_id = os.environ.get("BUILD_BUILDID") pipeline_name = os.environ.get("TRAINING_PIPELINE_NAME") epis_datastore = os.environ.get("EPIS_DATASTORE") epis_container = os.environ.get("EPIS_CONTAINER") aml_workspace = get_workspace( workspace_name, resource_group, subscription_id, tenant_id, app_id, app_secret) print(aml_workspace) aml_compute = get_compute( aml_workspace, compute_name, vm_size) if aml_compute is not None: print(aml_compute) run_config = RunConfiguration(conda_dependencies=CondaDependencies.create( conda_packages=['numpy==1.18.1', 'pandas', 'tensorflow-gpu==2.0.0'], pip_packages=['azure', 'azureml-core==1.0.60', 'azureml-tensorboard', 'azure-storage==0.36.0', 'tqdm==4.41.1', 'opencv-python==4.1.2.30', 'easydict==1.9', 'matplotlib==3.1.3']) ) run_config.environment.docker.enabled = True run_config.environment.docker.gpu_support = True run_config.environment.docker.base_image = DEFAULT_GPU_IMAGE model_name = PipelineParameter( name="model_name", default_value=model_name) release_id = PipelineParameter( name="release_id", default_value=build_id) train_step = PythonScriptStep( name="Train Model", script_name=train_script_path, compute_target=aml_compute, source_directory=sources_directory_train, arguments=[ "--release_id", release_id, "--model_name", model_name, "--ckpt_path", ckpt_path, "--datastore", epis_datastore, "--storage_container", epis_container, ], runconfig=run_config, allow_reuse=False, ) print("Step Train created") evaluate_step = PythonScriptStep( name="Evaluate Model", script_name=evaluate_script_path, compute_target=aml_compute, source_directory=sources_directory_train, arguments=[ "--release_id", release_id, "--model_name", model_name, "--ckpt_path", ckpt_path, "--datastore", epis_datastore, "--storage_container", epis_container, ], runconfig=run_config, allow_reuse=False, ) print("Step Evaluate created") generate_report_step = PythonScriptStep( name="Generate Report Model", script_name=generate_report_name, compute_target=aml_compute, source_directory=generate_report_path, arguments=[ "--release_id", release_id, "--model_name", model_name, "--ckpt_path", ckpt_path, "--datastore", epis_datastore, "--storage_container", epis_container, ], runconfig=run_config, allow_reuse=False, ) print("Step generate report created") evaluate_step.run_after(train_step) generate_report_step.run_after(evaluate_step) steps = [train_step, evaluate_step, generate_report_step] train_pipeline = Pipeline(workspace=aml_workspace, steps=steps) train_pipeline.validate() published_pipeline = train_pipeline.publish( name=pipeline_name, description="Model training/retraining pipeline", version=build_id ) print(f'Published pipeline: {published_pipeline.name}') print(f'for build {published_pipeline.version}')
data_factory_compute = DataFactoryCompute(ws, data_factory_name) aml_compute = AmlCompute(ws, aml_compute_target) # We explicitly declare the data we're using in this training pipeline source_images = DataReference(datastore=source_ds, data_reference_name="original_images", path_on_datastore=default_dataset) dest_images = DataReference(datastore=ds, data_reference_name="transferred_images", path_on_datastore='training_images') training_dataset = DataPath(datastore=source_ds, path_on_datastore=default_dataset) # Parameters make it easy for us to re-run this training pipeline, including for retraining. model_variant = PipelineParameter(name="model_variant", default_value='sodacans') training_dataset_param = (PipelineParameter(name="training_dataset", default_value=training_dataset), DataPathComputeBinding()) # Copying data into a datastore we manage ensures we can reproduce the model later on. datatransfer = DataTransferStep( name= "Copy training data for improved performance and model reproducibility", source_data_reference=source_images, destination_data_reference=dest_images, compute_target=data_factory_compute) # We pass the trained model from the transfer learning step to the model registration step model = PipelineData(name="model", datastore=ds,
def main(): e = Env() print(e.__dict__) # Get Azure machine learning workspace aml_workspace = Workspace.get(name=e.workspace_name, subscription_id=e.subscription_id, resource_group=e.resource_group) print("get_workspace:") print(aml_workspace) # Get Azure machine learning cluster aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size) if aml_compute is not None: print("aml_compute:") print(aml_compute) # Create a reusable Azure ML environment environment = get_environment(aml_workspace, e.aml_env_name, create_new=e.rebuild_env) # run_config = RunConfiguration() run_config.environment = environment if e.datastore_name: datastore_name = e.datastore_name else: datastore_name = aml_workspace.get_default_datastore().name run_config.environment.environment_variables[ "DATASTORE_NAME"] = datastore_name # NOQA: E501 model_name_param = PipelineParameter(name="model_name", default_value=e.model_name) dataset_version_param = PipelineParameter(name="dataset_version", default_value=e.dataset_version) data_file_path_param = PipelineParameter(name="data_file_path", default_value="none") caller_run_id_param = PipelineParameter(name="caller_run_id", default_value="none") # Get dataset name dataset_name = e.dataset_name # Check to see if dataset exists if dataset_name not in aml_workspace.datasets: raise ValueError( f"can't find dataset {dataset_name} in datastore {datastore_name}") # Create PipelineData to pass data between steps model_data = PipelineData("model_data", datastore=aml_workspace.get_default_datastore()) train_ds = (PipelineData("train_ds", datastore=aml_workspace.get_default_datastore()). as_dataset().parse_delimited_files().register( name="train", create_new_version=True)) test_ds = (PipelineData( "test_ds", datastore=aml_workspace.get_default_datastore()).as_dataset( ).parse_delimited_files().register(name="test", create_new_version=True)) prepare_step = PythonScriptStep( name="Prepare Data", script_name=e.prepare_script_path, compute_target=aml_compute, source_directory=e.sources_directory_train, outputs=[train_ds, test_ds], arguments=[ "--dataset_version", dataset_version_param, "--data_file_path", data_file_path_param, "--dataset_name", dataset_name, "--caller_run_id", caller_run_id_param, "--train_ds", train_ds, "--test_ds", test_ds ], runconfig=run_config, allow_reuse=True, ) print("Step Prepare created") train_step = PythonScriptStep( name="Train Model", script_name=e.train_script_path, compute_target=aml_compute, source_directory=e.sources_directory_train, inputs=[ train_ds.as_named_input("training_data"), test_ds.as_named_input("testing_data") ], outputs=[model_data], arguments=[ "--model_name", model_name_param, "--model_data", model_data ], runconfig=run_config, allow_reuse=False, ) print("Step Train created") evaluate_step = PythonScriptStep( name="Evaluate Model ", script_name=e.evaluate_script_path, compute_target=aml_compute, source_directory=e.sources_directory_train, arguments=[ "--model_name", model_name_param, "--allow_run_cancel", e.allow_run_cancel, ], runconfig=run_config, allow_reuse=False, ) print("Step Evaluate created") register_step = PythonScriptStep( name="Register Model ", script_name=e.register_script_path, compute_target=aml_compute, source_directory=e.sources_directory_train, inputs=[model_data], arguments=[ "--model_name", model_name_param, "--step_input", model_data ], runconfig=run_config, allow_reuse=False, ) print("Step Register created") # Check run_evaluation flag to include or exclude evaluation step. if (e.run_evaluation).lower() == "true": print("Include evaluation step before register step.") evaluate_step.run_after(train_step) register_step.run_after(evaluate_step) steps = [prepare_step, train_step, evaluate_step, register_step] else: print("Exclude evaluation step and directly run register step.") register_step.run_after(train_step) steps = [prepare_step, train_step, register_step] train_pipeline = Pipeline(workspace=aml_workspace, steps=steps) train_pipeline._set_experiment_name train_pipeline.validate() published_pipeline = train_pipeline.publish( name=e.pipeline_name, description="Model training/retraining pipeline", version=e.build_id) print(f"Published pipeline: {published_pipeline.name}") print(f"for build {published_pipeline.version}")
def main(): e = Env() # Get Azure machine learning workspace aml_workspace = Workspace.get( name=e.workspace_name, subscription_id=e.subscription_id, resource_group=e.resource_group, ) print(f"get_workspace:{aml_workspace}") # Get Azure machine learning cluster aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size) if aml_compute is not None: print(f"aml_compute:{aml_compute}") # Create a reusable Azure ML environment environment = get_environment( aml_workspace, e.aml_env_name, conda_dependencies_file=e.aml_env_train_conda_dep_file, create_new=e.rebuild_env, ) # run_config = RunConfiguration() run_config.environment = environment if e.datastore_name: datastore_name = e.datastore_name else: datastore_name = aml_workspace.get_default_datastore().name run_config.environment.environment_variables[ "DATASTORE_NAME"] = datastore_name # NOQA: E501 # datastore and dataset names are fixed for this pipeline, however # data_file_path can be specified for registering new versions of dataset # Note that AML pipeline parameters don't take empty string as default, "" won't work # NOQA: E501 model_name_param = PipelineParameter( name="model_name", default_value=e.model_name) # NOQA: E501 data_file_path_param = PipelineParameter( name="data_file_path", default_value="nopath") # NOQA: E501 ml_params = PipelineParameter(name="ml_params", default_value="default") # NOQA: E501 # Create a PipelineData to pass data between steps pipeline_data = PipelineData( "pipeline_data", datastore=aml_workspace.get_default_datastore()) train_step = PythonScriptStep( name="Train Model", script_name="train/train_aml.py", compute_target=aml_compute, source_directory=e.sources_directory_train, outputs=[pipeline_data], arguments=[ "--model_name", model_name_param, "--step_output", pipeline_data, "--data_file_path", data_file_path_param, "--dataset_name", e.processed_dataset_name, "--datastore_name", datastore_name, "--ml_params", ml_params, ], runconfig=run_config, allow_reuse=True, ) print("Step Train created") evaluate_step = PythonScriptStep( name="Evaluate Model ", script_name="evaluate/evaluate_model.py", compute_target=aml_compute, source_directory=e.sources_directory_train, arguments=[ "--model_name", model_name_param, "--ml_params", ml_params, ], runconfig=run_config, allow_reuse=False, ) print("Step Evaluate created") register_step = PythonScriptStep( name="Register Model ", script_name="register/register_model.py", compute_target=aml_compute, source_directory=e.sources_directory_train, inputs=[pipeline_data], arguments=[ "--model_name", model_name_param, "--step_input", pipeline_data, "--ml_params", ml_params, ], runconfig=run_config, allow_reuse=False, ) print("Step Register created") evaluate_step.run_after(train_step) register_step.run_after(evaluate_step) steps = [train_step, evaluate_step, register_step] train_pipeline = Pipeline(workspace=aml_workspace, steps=steps) train_pipeline._set_experiment_name train_pipeline.validate() published_pipeline = train_pipeline.publish( name=e.training_pipeline_name, description="Model training/retraining pipeline", version=e.build_id, ) print(f"Published pipeline: {published_pipeline.name}") print(f"for build {published_pipeline.version}")
def main(): e = Env() # Get Azure machine learning workspace aml_workspace = Workspace.get(name=e.workspace_name, subscription_id=e.subscription_id, resource_group=e.resource_group) print("get_workspace:") print(aml_workspace) # Get Azure machine learning cluster aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size) if aml_compute is not None: print("aml_compute:") print(aml_compute) # Create a reusable run configuration environment # Read definition from diabetes_regression/azureml_environment.json environment = Environment.load_from_directory(e.sources_directory_train) if (e.collection_uri is not None and e.teamproject_name is not None): builduri_base = e.collection_uri + e.teamproject_name builduri_base = builduri_base + "/_build/results?buildId=" environment.environment_variables["BUILDURI_BASE"] = builduri_base environment.register(aml_workspace) run_config = RunConfiguration() run_config.environment = environment model_name_param = PipelineParameter(name="model_name", default_value=e.model_name) build_id_param = PipelineParameter(name="build_id", default_value=e.build_id) # Get dataset name dataset_name = e.dataset_name # Check to see if dataset exists if (dataset_name not in aml_workspace.datasets): # Create dataset from diabetes sample data sample_data = load_diabetes() df = pd.DataFrame(data=sample_data.data, columns=sample_data.feature_names) df['Y'] = sample_data.target file_name = 'diabetes.csv' df.to_csv(file_name, index=False) # Upload file to default datastore in workspace default_ds = aml_workspace.get_default_datastore() target_path = 'training-data/' default_ds.upload_files(files=[file_name], target_path=target_path, overwrite=True, show_progress=False) # Register dataset path_on_datastore = os.path.join(target_path, file_name) dataset = Dataset.Tabular.from_delimited_files( path=(default_ds, path_on_datastore)) dataset = dataset.register(workspace=aml_workspace, name=dataset_name, description='diabetes training data', tags={'format': 'CSV'}, create_new_version=True) # Get the dataset dataset = Dataset.get_by_name(aml_workspace, dataset_name) # Create a PipelineData to pass data between steps pipeline_data = PipelineData( 'pipeline_data', datastore=aml_workspace.get_default_datastore()) train_step = PythonScriptStep( name="Train Model", script_name=e.train_script_path, compute_target=aml_compute, source_directory=e.sources_directory_train, inputs=[dataset.as_named_input('training_data')], outputs=[pipeline_data], arguments=[ "--build_id", build_id_param, "--model_name", model_name_param, "--step_output", pipeline_data ], runconfig=run_config, allow_reuse=False, ) print("Step Train created") evaluate_step = PythonScriptStep( name="Evaluate Model ", script_name=e.evaluate_script_path, compute_target=aml_compute, source_directory=e.sources_directory_train, arguments=[ "--build_id", build_id_param, "--model_name", model_name_param, "--allow_run_cancel", e.allow_run_cancel, ], runconfig=run_config, allow_reuse=False, ) print("Step Evaluate created") register_step = PythonScriptStep( name="Register Model ", script_name=e.register_script_path, compute_target=aml_compute, source_directory=e.sources_directory_train, inputs=[pipeline_data], arguments=[ "--build_id", build_id_param, "--model_name", model_name_param, "--step_input", pipeline_data, ], runconfig=run_config, allow_reuse=False, ) print("Step Register created") # Check run_evaluation flag to include or exclude evaluation step. if ((e.run_evaluation).lower() == 'true'): print("Include evaluation step before register step.") evaluate_step.run_after(train_step) register_step.run_after(evaluate_step) steps = [train_step, evaluate_step, register_step] else: print("Exclude evaluation step and directly run register step.") register_step.run_after(train_step) steps = [train_step, register_step] train_pipeline = Pipeline(workspace=aml_workspace, steps=steps) train_pipeline._set_experiment_name train_pipeline.validate() published_pipeline = train_pipeline.publish( name=e.pipeline_name, description="Model training/retraining pipeline", version=e.build_id) print(f'Published pipeline: {published_pipeline.name}') print(f'for build {published_pipeline.version}')
def __init__(self, request_id: str, use_url: bool, input_container_sas: Optional[str], internal_datastore: Mapping[str, str], model_name: str): try: self.request_id = request_id aml_config = api_config.AML_CONFIG self.ws = Workspace( subscription_id=aml_config['subscription_id'], resource_group=aml_config['resource_group'], workspace_name=aml_config['workspace_name'], auth=svc_pr ) print('AMLCompute constructor, AML workspace obtained.') internal_dir, output_dir = self._get_data_references( request_id, internal_datastore) aml_compute_name = aml_config['aml_compute_name'] compute_target = self.ws.compute_targets[aml_compute_name] pip_packages = ['tensorflow-gpu==1.12.0', 'pillow', 'numpy', 'azure-storage-blob==2.1.0', 'azureml-defaults==1.0.41'] dependencies = CondaDependencies.create(pip_packages=pip_packages) amlcompute_run_config = RunConfiguration( conda_dependencies=dependencies) amlcompute_run_config.environment.docker.enabled = True amlcompute_run_config.environment.docker.gpu_support = True amlcompute_run_config.environment.docker.base_image = DEFAULT_GPU_IMAGE # pylint: disable=line-too-long amlcompute_run_config.environment.spark.precache_packages = False # default values are required and need to be literal values or data # references as JSON param_job_id = PipelineParameter( name='param_job_id', default_value='default_job_id') param_begin_index = PipelineParameter( name='param_begin_index', default_value=0) param_end_index = PipelineParameter( name='param_end_index', default_value=0) param_detection_threshold = PipelineParameter( name='param_detection_threshold', default_value=0.05) batch_score_step = PythonScriptStep( aml_config['script_name'], source_directory=aml_config['source_dir'], hash_paths=['.'], # include all contents of source_directory name='batch_scoring', arguments=[ '--job_id', param_job_id, '--request_id', request_id, '--model_name', model_name, '--input_container_sas', input_container_sas, # can be None '--use_url', use_url, '--internal_dir', internal_dir, '--begin_index', param_begin_index, # inclusive '--end_index', param_end_index, # exclusive '--output_dir', output_dir, '--detection_threshold', param_detection_threshold], compute_target=compute_target, inputs=[internal_dir], outputs=[output_dir], runconfig=amlcompute_run_config) self.pipeline = Pipeline( workspace=self.ws, steps=[batch_score_step]) self.aml_config = aml_config print('AMLCompute constructor all good.') except Exception as e: raise RuntimeError('Error setting up AML Compute resource: {}.'.format(e))
pipeline_run1 = Experiment(ws, 'Titanic_Pipeline_Notebook').submit(pipeline1) pipeline_run1.wait_for_completion() # RunDetails(pipeline_run1).show() step_runs = pipeline_run1.get_children() for step_run in step_runs: status = step_run.get_status() print('Script:', step_run.name, 'status:', status) # Change this if you want to see details even if the Step has succeeded. if status == "Failed": joblog = step_run.get_job_log() print('job log:', joblog) pipeline_param = PipelineParameter(name="pipeline_arg", default_value=10) published_pipeline1 = pipeline_run1.publish_pipeline( name="Published_Titanic_Pipeline_Notebook", description="Titanic_Pipeline_Notebook Published Pipeline Description", version="1.0") from azureml.pipeline.core import PublishedPipeline import requests response = requests.post(published_pipeline1.endpoint, json={ "ExperimentName": "Titanic_Pipeline_Notebook", "ParameterAssignments": { "pipeline_arg": 20 }
def main(): e = Env() aml_workspace = Workspace.get( name=e.workspace_name, subscription_id=e.subscription_id, resource_group=e.resource_group ) print("get_workspace:") print(aml_workspace) aml_compute = get_compute( aml_workspace, e.compute_name, e.vm_size) if aml_compute is not None: print("aml_compute:") print(aml_compute) environment = get_environment( aml_workspace, e.aml_env_name, create_new=e.rebuild_env) run_config = RunConfiguration() run_config.environment = environment if (e.datastore_name): datastore_name = e.datastore_name else: datastore_name = aml_workspace.get_default_datastore().name run_config.environment.environment_variables["DATASTORE_NAME"] \ = datastore_name dataset_name = e.dataset_name file_name = e.file_name datastore = Datastore.get(aml_workspace, datastore_name) if (dataset_name not in aml_workspace.datasets): raise Exception("Could not find dataset at \"%s\"." % dataset_name) else: dataset = Dataset.get_by_name(aml_workspace, name=dataset_name) dataset.download(target_path='.', overwrite=True) datastore.upload_files([file_name], target_path=dataset_name, overwrite=True) raw_data_file = DataReference(datastore=datastore, data_reference_name="Raw_Data_File", path_on_datastore=dataset_name + '/' + file_name) clean_data_file = PipelineParameter(name="clean_data_file", default_value="/clean_data.csv") clean_data_folder = PipelineData("clean_data_folder", datastore=datastore) prepDataStep = PythonScriptStep(name="Prepare Data", source_directory=e.sources_directory_train, script_name=e.data_prep_script_path, arguments=["--raw_data_file", raw_data_file, "--clean_data_folder", clean_data_folder, "--clean_data_file", clean_data_file], inputs=[raw_data_file], outputs=[clean_data_folder], compute_target=aml_compute, allow_reuse=False) print("Step Prepare Data created") new_model_file = PipelineParameter(name="new_model_file ", default_value='/' + e.model_name + '.pkl') new_model_folder = PipelineData("new_model_folder", datastore=datastore) est = SKLearn(source_directory=e.sources_directory_train, entry_script=e.train_script_path, pip_packages=['azureml-sdk', 'scikit-learn==0.20.3', 'azureml-dataprep[pandas,fuse]>=1.1.14'], compute_target=aml_compute) trainingStep = EstimatorStep( name="Model Training", estimator=est, estimator_entry_script_arguments=["--clean_data_folder", clean_data_folder, "--new_model_folder", new_model_folder, "--clean_data_file", clean_data_file.default_value, "--new_model_file", new_model_file.default_value], runconfig_pipeline_params=None, inputs=[clean_data_folder], outputs=[new_model_folder], compute_target=aml_compute, allow_reuse=False) print("Step Train created") model_name_param = PipelineParameter(name="model_name", default_value=e.model_name) evaluateStep = PythonScriptStep( name="Evaluate Model", source_directory=e.sources_directory_train, script_name=e.evaluate_script_path, arguments=["--model_name", model_name_param], compute_target=aml_compute, allow_reuse=False) print("Step Evaluate created") registerStep = PythonScriptStep( name="Register Model", source_directory=e.sources_directory_train, script_name=e.register_script_path, arguments=["--new_model_folder", new_model_folder, "--new_model_file", new_model_file, "--model_name", model_name_param], inputs=[new_model_folder], compute_target=aml_compute, allow_reuse=False) print("Step Register created") if ((e.run_evaluation).lower() == 'true'): print("Include evaluation step before register step.") trainingStep.run_after(prepDataStep) evaluateStep.run_after(trainingStep) registerStep.run_after(evaluateStep) else: print("Exclude evaluation step and directly run register step.") trainingStep.run_after(prepDataStep) registerStep.run_after(trainingStep) pipeline = Pipeline(workspace=aml_workspace, steps=[registerStep]) pipeline.validate() print("Pipeline is built") pipeline._set_experiment_name published_pipeline = pipeline.publish( name=e.pipeline_name, description="Predict Employee Retention Model training pipeline", version=e.build_id ) print(f'Published pipeline: {published_pipeline.name}') print(f'for build {published_pipeline.version}')
def main(): e = Env() # Get Azure machine learning workspace aml_workspace = get_workspace(e.workspace_name, e.resource_group, e.subscription_id, e.tenant_id, e.app_id, e.app_secret) print("get_workspace:") print(aml_workspace) # Get Azure machine learning cluster aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size) if aml_compute is not None: print("aml_compute:") print(aml_compute) run_config = RunConfiguration(conda_dependencies=CondaDependencies.create( conda_packages=[ 'numpy', 'pandas', 'scikit-learn', 'tensorflow', 'keras' ], pip_packages=[ 'azure', 'azureml-core', 'azure-storage', 'azure-storage-blob' ])) run_config.environment.docker.enabled = True model_name = PipelineParameter(name="model_name", default_value=e.model_name) release_id = PipelineParameter(name="release_id", default_value="0") train_step = PythonScriptStep( name="Train Model", script_name=e.train_script_path, compute_target=aml_compute, source_directory=e.sources_directory_train, arguments=[ "--release_id", release_id, "--model_name", model_name, ], runconfig=run_config, allow_reuse=False, ) print("Step Train created") evaluate_step = PythonScriptStep( name="Evaluate Model ", script_name=e.evaluate_script_path, compute_target=aml_compute, source_directory=e.sources_directory_train, arguments=[ "--release_id", release_id, "--model_name", model_name, ], runconfig=run_config, allow_reuse=False, ) print("Step Evaluate created") evaluate_step.run_after(train_step) steps = [evaluate_step] train_pipeline = Pipeline(workspace=aml_workspace, steps=steps) train_pipeline.validate() published_pipeline = train_pipeline.publish( name=e.pipeline_name, description="Model training/retraining pipeline", version=e.build_id) print(f'Published pipeline: {published_pipeline.name}') print(f'for build {published_pipeline.version}')
def main(): e = Env() # Get Azure machine learning workspace aml_workspace = Workspace.get(name=e.workspace_name, subscription_id=e.subscription_id, resource_group=e.resource_group) print("get_workspace:") print(aml_workspace) # Get Azure machine learning cluster aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size) if aml_compute is not None: print("aml_compute:") print(aml_compute) # Create a run configuration environment conda_deps_file = "diabetes_regression/training_dependencies.yml" conda_deps = CondaDependencies(conda_deps_file) run_config = RunConfiguration(conda_dependencies=conda_deps) run_config.environment.docker.enabled = True config_envvar = {} if (e.collection_uri is not None and e.teamproject_name is not None): builduri_base = e.collection_uri + e.teamproject_name builduri_base = builduri_base + "/_build/results?buildId=" config_envvar["BUILDURI_BASE"] = builduri_base run_config.environment.environment_variables = config_envvar model_name_param = PipelineParameter(name="model_name", default_value=e.model_name) build_id_param = PipelineParameter(name="build_id", default_value=e.build_id) dataset_name = "" if (e.datastore_name is not None and e.datafile_name is not None): dataset_name = e.dataset_name datastore = Datastore.get(aml_workspace, e.datastore_name) data_path = [(datastore, e.datafile_name)] dataset = Dataset.Tabular.from_delimited_files(path=data_path) dataset.register(workspace=aml_workspace, name=e.dataset_name, description="dataset with training data", create_new_version=True) train_step = PythonScriptStep( name="Train Model", script_name=e.train_script_path, compute_target=aml_compute, source_directory=e.sources_directory_train, arguments=[ "--build_id", build_id_param, "--model_name", model_name_param, "--dataset_name", dataset_name, ], runconfig=run_config, allow_reuse=False, ) print("Step Train created") evaluate_step = PythonScriptStep( name="Evaluate Model ", script_name=e.evaluate_script_path, compute_target=aml_compute, source_directory=e.sources_directory_train, arguments=[ "--build_id", build_id_param, "--model_name", model_name_param, "--allow_run_cancel", e.allow_run_cancel, ], runconfig=run_config, allow_reuse=False, ) print("Step Evaluate created") register_step = PythonScriptStep( name="Register Model ", script_name=e.register_script_path, compute_target=aml_compute, source_directory=e.sources_directory_train, arguments=[ "--build_id", build_id_param, "--model_name", model_name_param, ], runconfig=run_config, allow_reuse=False, ) print("Step Register created") # Check run_evaluation flag to include or exclude evaluation step. if ((e.run_evaluation).lower() == 'true'): print("Include evaluation step before register step.") evaluate_step.run_after(train_step) register_step.run_after(evaluate_step) steps = [train_step, evaluate_step, register_step] else: print("Exclude evaluation step and directly run register step.") register_step.run_after(train_step) steps = [train_step, register_step] train_pipeline = Pipeline(workspace=aml_workspace, steps=steps) train_pipeline._set_experiment_name train_pipeline.validate() published_pipeline = train_pipeline.publish( name=e.pipeline_name, description="Model training/retraining pipeline", version=e.build_id) print(f'Published pipeline: {published_pipeline.name}') print(f'for build {published_pipeline.version}')
def main(): e = Env() # Get Azure machine learning workspace aml_workspace = Workspace.get(name=e.workspace_name, subscription_id=e.subscription_id, resource_group=e.resource_group) print("get_workspace:") print(aml_workspace) # Get Azure machine learning cluster aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size) if aml_compute is not None: print("aml_compute:") print(aml_compute) run_config = RunConfiguration(conda_dependencies=CondaDependencies.create( conda_packages=[ 'numpy', 'pandas', 'scikit-learn', 'tensorflow', 'keras' ], pip_packages=[ 'azure', 'azureml-core', 'azure-storage', 'azure-storage-blob' ])) run_config.environment.docker.enabled = True config_envvar = {} if (e.collection_uri is not None and e.teamproject_name is not None): builduri_base = e.collection_uri + e.teamproject_name builduri_base = builduri_base + "/_build/results?buildId=" config_envvar["BUILDURI_BASE"] = builduri_base run_config.environment.environment_variables = config_envvar model_name_param = PipelineParameter(name="model_name", default_value=e.model_name) build_id_param = PipelineParameter(name="build_id", default_value=e.build_id) hyperparameter_alpha_param = PipelineParameter(name="hyperparameter_alpha", default_value=0.5) train_step = PythonScriptStep( name="Train Model", script_name=e.train_script_path, compute_target=aml_compute, source_directory=e.sources_directory_train, arguments=[ "--build_id", build_id_param, "--model_name", model_name_param, "--alpha", hyperparameter_alpha_param, ], runconfig=run_config, allow_reuse=False, ) print("Step Train created") evaluate_step = PythonScriptStep( name="Evaluate Model ", script_name=e.evaluate_script_path, compute_target=aml_compute, source_directory=e.sources_directory_train, arguments=[ "--build_id", build_id_param, "--model_name", model_name_param, ], runconfig=run_config, allow_reuse=False, ) print("Step Evaluate created") register_step = PythonScriptStep( name="Register Model ", script_name=e.register_script_path, compute_target=aml_compute, source_directory=e.sources_directory_train, arguments=[ "--build_id", build_id_param, "--model_name", model_name_param, ], runconfig=run_config, allow_reuse=False, ) print("Step Register created") evaluate_step.run_after(train_step) register_step.run_after(evaluate_step) steps = [train_step, evaluate_step, register_step] train_pipeline = Pipeline(workspace=aml_workspace, steps=steps) train_pipeline._set_experiment_name train_pipeline.validate() published_pipeline = train_pipeline.publish( name=e.pipeline_name, description="Model training/retraining pipeline", version=e.build_id) print(f'Published pipeline: {published_pipeline.name}') print(f'for build {published_pipeline.version}')
def __init__(self, request_id, input_container_sas, internal_datastore): try: aml_config = api_config.AML_CONFIG self.ws = Workspace(subscription_id=aml_config['subscription_id'], resource_group=aml_config['resource_group'], workspace_name=aml_config['workspace_name'], auth=svc_pr) print('AMLCompute constructor, AML workspace obtained.') internal_dir, output_dir = self._get_data_references( request_id, internal_datastore) compute_target = self.ws.compute_targets[ aml_config['aml_compute_name']] dependencies = CondaDependencies.create(pip_packages=[ 'tensorflow-gpu==1.9.0', 'pillow', 'numpy', 'azure', 'azure-storage-blob', 'azureml-defaults' ]) amlcompute_run_config = RunConfiguration( conda_dependencies=dependencies) amlcompute_run_config.environment.docker.enabled = True amlcompute_run_config.environment.docker.gpu_support = True amlcompute_run_config.environment.docker.base_image = DEFAULT_GPU_IMAGE amlcompute_run_config.environment.spark.precache_packages = False # default values are required and need to be literal values or data references as JSON param_job_id = PipelineParameter(name='param_job_id', default_value='default_job_id') param_begin_index = PipelineParameter(name='param_begin_index', default_value=0) param_end_index = PipelineParameter(name='param_end_index', default_value=0) param_detection_threshold = PipelineParameter( name='param_detection_threshold', default_value=0.05) param_batch_size = PipelineParameter(name='param_batch_size', default_value=8) batch_score_step = PythonScriptStep( aml_config['script_name'], source_directory=aml_config['source_dir'], name='batch_scoring', arguments=[ '--job_id', param_job_id, '--model_name', aml_config['model_name'], '--input_container_sas', input_container_sas, '--internal_dir', internal_dir, '--begin_index', param_begin_index, # inclusive '--end_index', param_end_index, # exclusive '--output_dir', output_dir, '--detection_threshold', param_detection_threshold, '--batch_size', param_batch_size ], compute_target=compute_target, inputs=[internal_dir], outputs=[output_dir], runconfig=amlcompute_run_config) self.pipeline = Pipeline(workspace=self.ws, steps=[batch_score_step]) self.aml_config = aml_config print('AMLCompute constructor all good.') except Exception as e: raise RuntimeError( 'Error in setting up AML Compute resource: {}.'.format(str(e)))
create_or_update_schedule, get_or_create_compute, get_workspace, publish_pipeline, ) from config import ml_pipeline_test_config as config ws = get_workspace(config) compute_target = get_or_create_compute(ws, **config["compute"]) ### # Define and set up pipeline ### pipeline_param = PipelineParameter(name="my_arg", default_value="default") my_step = PythonScriptStep( name="My Script Step", script_name="scriptstep.py", arguments=[pipeline_param], inputs=[], outputs=[], compute_target=compute_target, source_directory="src", allow_reuse=True, runconfig=RunConfiguration(conda_dependencies=CondaDependencies( conda_dependencies_file_path="environment.yml")), ) pipeline_id, pipeline_endpoint = publish_pipeline(ws, [my_step], "blabla")
args = parser.parse_args() # Get workspace ws = Workspace.from_config(path="aml_config/config.json", auth=cli_auth) def_blob_store = Datastore(ws, "workspaceblobstore") # Get AML Compute name and Experiment Name with open("aml_config/security_config.json") as f: config = json.load(f) experiment_name = config["experiment_name"] aml_cluster_name = config["aml_cluster_name"] aml_pipeline_name = "training-pipeline" #model_name = config["model_name"] model_name = PipelineParameter(name="model_name", default_value="sklearn_regression_model.pkl") source_directory = "code" # Run Config # Declare packages dependencies required in the pipeline (these can also be expressed as a YML file) # cd = CondaDependencies.create(pip_packages=["azureml-defaults", 'tensorflow==1.8.0']) cd = CondaDependencies("aml_config/conda_dependencies.yml") run_config = RunConfiguration(conda_dependencies=cd) aml_compute = ws.compute_targets[aml_cluster_name] run_config.environment.docker.enabled = True run_config.environment.spark.precache_packages = False jsonconfigs = PipelineData("jsonconfigs", datastore=def_blob_store)
def main(): e = Env() # Get Azure machine learning workspace aml_workspace = Workspace.get(name=e.workspace_name, subscription_id=e.subscription_id, resource_group=e.resource_group) print("get_workspace:") print(aml_workspace) # Get Azure machine learning cluster aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size) if aml_compute is not None: print("aml_compute:") print(aml_compute) # Create a reusable Azure ML environment environment = get_environment(aml_workspace, e.aml_env_name, create_new=False) # NOQA: E501 run_config = RunConfiguration() run_config.environment = environment if (e.datastore_name): datastore_name = e.datastore_name else: datastore_name = aml_workspace.get_default_datastore().name run_config.environment.environment_variables[ "DATASTORE_NAME"] = datastore_name # NOQA: E501 model_name_param = PipelineParameter(name="model_name", default_value=e.model_name) dataset_version_param = PipelineParameter(name="dataset_version", default_value=e.dataset_version) data_file_path_param = PipelineParameter(name="data_file_path", default_value="none") caller_run_id_param = PipelineParameter(name="caller_run_id", default_value="none") # Get dataset name dataset_name = e.dataset_name # # Check to see if dataset exists # if (dataset_name not in aml_workspace.datasets): # # Create dataset from lacemlops sample data # sample_data = load_lacemlops() # df = pd.DataFrame( # data=sample_data.data, # columns=sample_data.feature_names) # df['Y'] = sample_data.target # file_name = 'lacemlops.csv' # df.to_csv(file_name, index=False) # # Upload file to default datastore in workspace # datatstore = Datastore.get(aml_workspace, datastore_name) # target_path = 'training-data/' # datatstore.upload_files( # files=[file_name], # target_path=target_path, # overwrite=True, # show_progress=False) # # Register dataset # path_on_datastore = os.path.join(target_path, file_name) # dataset = Dataset.Tabular.from_delimited_files( # path=(datatstore, path_on_datastore)) # dataset = dataset.register( # workspace=aml_workspace, # name=dataset_name, # description='lacemlops training data', # tags={'format': 'CSV'}, # create_new_version=True) # Create a PipelineData to pass data between steps pipeline_data = PipelineData( 'pipeline_data', datastore=aml_workspace.get_default_datastore()) train_step = PythonScriptStep( name="Train Model", script_name=e.train_script_path, compute_target=aml_compute, source_directory=e.sources_directory_train, outputs=[pipeline_data], arguments=[ "--model_name", model_name_param, "--step_output", pipeline_data, "--dataset_version", dataset_version_param, "--data_file_path", data_file_path_param, "--caller_run_id", caller_run_id_param, "--dataset_name", dataset_name, ], runconfig=run_config, allow_reuse=False, ) print("Step Train created") evaluate_step = PythonScriptStep( name="Evaluate Model ", script_name=e.evaluate_script_path, compute_target=aml_compute, source_directory=e.sources_directory_train, arguments=[ "--model_name", model_name_param, "--allow_run_cancel", e.allow_run_cancel, ], runconfig=run_config, allow_reuse=False, ) print("Step Evaluate created") register_step = PythonScriptStep( name="Register Model ", script_name=e.register_script_path, compute_target=aml_compute, source_directory=e.sources_directory_train, inputs=[pipeline_data], arguments=[ "--model_name", model_name_param, "--step_input", pipeline_data, ], runconfig=run_config, allow_reuse=False, ) print("Step Register created") # Check run_evaluation flag to include or exclude evaluation step. if ((e.run_evaluation).lower() == 'true'): print("Include evaluation step before register step.") evaluate_step.run_after(train_step) register_step.run_after(evaluate_step) steps = [train_step, evaluate_step, register_step] else: print("Exclude evaluation step and directly run register step.") register_step.run_after(train_step) steps = [train_step, register_step] train_pipeline = Pipeline(workspace=aml_workspace, steps=steps) train_pipeline._set_experiment_name train_pipeline.validate() published_pipeline = train_pipeline.publish( name=e.pipeline_name, description="Model training/retraining pipeline", version=e.build_id) print(f'Published pipeline: {published_pipeline.name}') print(f'for build {published_pipeline.version}')
run_config = RunConfiguration() run_config.environment.docker.enabled = True run_config.environment.docker.base_image = DEFAULT_CPU_IMAGE run_config.environment.python.user_managed_dependencies = False pip_packages = [ "azureml-sdk==1.0.17", "scikit-learn==0.21.3", "download==0.3.4", "pandas==0.25.1", "spacy==2.1.4", "numpy==1.17.2" ] run_config.environment.python.conda_dependencies = CondaDependencies.create( pip_packages=pip_packages) pipeline_params = [] for k, v in vars(auth_params).items(): pipeline_params.append("--" + k) pipeline_params.append(PipelineParameter(name=k, default_value=v)) auth_params = pipeline_params.copy() pipeline_params += ["--processed_data_ref", processed_data_ref] pipeline_params += ["--input_data_ref", input_data_ref] process_step = PythonScriptStep(script_name="process.py", arguments=pipeline_params, inputs=[input_data_ref], outputs=[processed_data_ref], compute_target=compute_target_cpu, source_directory='./', runconfig=run_config) # step 2 cluster_name = "gpucluster" try:
def main(): load_dotenv() workspace_name = os.environ.get("BASE_NAME") + "-AML-WS" resource_group = os.environ.get("RESOURCE_GROUP") subscription_id = os.environ.get("SUBSCRIPTION_ID") tenant_id = os.environ.get("TENANT_ID") app_id = os.environ.get("SP_APP_ID") app_secret = os.environ.get("SP_APP_SECRET") deploy_script_path = os.environ.get("DEPLOY_SCRIPT_PATH") vm_size = os.environ.get("AML_COMPUTE_CLUSTER_CPU_SKU") compute_name = os.environ.get("AML_COMPUTE_CLUSTER_NAME") model_name = os.environ.get("MODEL_NAME") build_id = os.environ.get("BUILD_BUILDID") pipeline_name = os.environ.get("DEPLOY_PIPELINE_NAME") service_name = os.environ.get("DEPLOY_SERVICE_NAME") sources_directory_train = os.environ.get("SOURCES_DIR_TRAIN") # Get Azure machine learning workspace aml_workspace = get_workspace(workspace_name, resource_group, subscription_id, tenant_id, app_id, app_secret) print(aml_workspace) # Get Azure machine learning cluster aml_compute = get_compute(aml_workspace, compute_name, vm_size) if aml_compute is not None: print(aml_compute) conda_dependencies = CondaDependencies.create( conda_packages=['numpy', 'pandas', 'scikit-learn'], pip_packages=[ 'azureml-core==1.0.72.*', 'azureml-sdk==1.0.72.*', 'azure-storage', 'azure-storage-blob', 'azureml-dataprep', 'azureml-datadrift==1.0.72.*' ], pin_sdk_version=False) print(conda_dependencies.serialize_to_string()) run_config = RunConfiguration(framework='Python', conda_dependencies=conda_dependencies) run_config.environment.docker.enabled = True model_name = PipelineParameter(name="model_name", default_value=model_name) print(model_name) release_id = PipelineParameter(name="release_id", default_value="0") print(release_id) service_name = PipelineParameter(name="service_name", default_value=service_name) print(service_name) deploy_step = PythonScriptStep( name="Deploy Model", script_name=deploy_script_path, compute_target=aml_compute, source_directory=sources_directory_train, arguments=[ "--release_id", release_id, "--model_name", model_name, "--service_name", service_name ], runconfig=run_config, allow_reuse=False, ) print("Step Deploy created") steps = [deploy_step] deploy_pipeline = Pipeline(workspace=aml_workspace, steps=steps) deploy_pipeline.validate() published_pipeline = deploy_pipeline.publish( name=pipeline_name, description="Model deploy pipeline", version=build_id) print(f'Published pipeline: {published_pipeline.name}') print(f'for build {published_pipeline.version}')
def __create_pipeline_parameter(self, pipeline_parameter_name, pipeline_parameter_value): return PipelineParameter(name=pipeline_parameter_name, default_value=pipeline_parameter_value)
def main(): load_dotenv() workspace_name = os.environ.get("BASE_NAME") + "-AML-WS" resource_group = os.environ.get("BASE_NAME") + "-AML-RG" subscription_id = os.environ.get("SUBSCRIPTION_ID") tenant_id = os.environ.get("TENANT_ID") app_id = os.environ.get("SP_APP_ID") app_secret = os.environ.get("SP_APP_SECRET") sources_directory_train = os.environ.get("SOURCES_DIR_TRAIN") train_script_path = os.environ.get("TRAIN_SCRIPT_PATH") evaluate_script_path = os.environ.get("EVALUATE_SCRIPT_PATH") register_script_path = os.environ.get("REGISTER_SCRIPT_PATH") vm_size_cpu = os.environ.get("AML_COMPUTE_CLUSTER_CPU_SKU") compute_name_cpu = os.environ.get("AML_COMPUTE_CLUSTER_NAME") model_name = os.environ.get("MODEL_NAME") # Get Azure machine learning workspace aml_workspace = get_workspace(workspace_name, resource_group, subscription_id, tenant_id, app_id, app_secret) print(aml_workspace) # Get Azure machine learning cluster aml_compute_cpu = get_compute(aml_workspace, compute_name_cpu, vm_size_cpu) if aml_compute_cpu is not None: print(aml_compute_cpu) run_config = RunConfiguration(conda_dependencies=CondaDependencies.create( conda_packages=[ 'numpy', 'pandas', 'scikit-learn', 'tensorflow', 'keras' ])) run_config.environment.docker.enabled = True model_name = PipelineParameter(name="model_name", default_value=model_name) def_blob_store = Datastore(aml_workspace, "workspaceblobstore") jsonconfigs = PipelineData("jsonconfigs", datastore=def_blob_store) config_suffix = datetime.datetime.now().strftime("%Y%m%d%H") train_step = PythonScriptStep( name="Train Model", script_name=train_script_path, compute_target=aml_compute_cpu, source_directory=sources_directory_train, arguments=[ "--config_suffix", config_suffix, "--json_config", jsonconfigs, "--model_name", model_name, ], runconfig=run_config, # inputs=[jsonconfigs], outputs=[jsonconfigs], allow_reuse=False, ) print("Step Train created") evaluate_step = PythonScriptStep( name="Evaluate Model ", script_name=evaluate_script_path, compute_target=aml_compute_cpu, source_directory=sources_directory_train, arguments=[ "--config_suffix", config_suffix, "--json_config", jsonconfigs, ], runconfig=run_config, inputs=[jsonconfigs], # outputs=[jsonconfigs], allow_reuse=False, ) print("Step Evaluate created") register_model_step = PythonScriptStep( name="Register New Trained Model", script_name=register_script_path, compute_target=aml_compute_cpu, source_directory=sources_directory_train, arguments=[ "--config_suffix", config_suffix, "--json_config", jsonconfigs, "--model_name", model_name, ], runconfig=run_config, inputs=[jsonconfigs], # outputs=[jsonconfigs], allow_reuse=False, ) print("Step register model created") evaluate_step.run_after(train_step) register_model_step.run_after(evaluate_step) steps = [register_model_step] train_pipeline = Pipeline(workspace=aml_workspace, steps=steps) train_pipeline.validate() published_pipeline = train_pipeline.publish( name="training-pipeline", description="Model training/retraining pipeline") train_pipeline_json = {} train_pipeline_json["rest_endpoint"] = published_pipeline.endpoint json_file_path = "ml_service/pipelines/train_pipeline.json" with open(json_file_path, "w") as outfile: json.dump(train_pipeline_json, outfile)
def main(): load_dotenv() workspace_name = os.environ.get("BASE_NAME") + "-AML-WS" resource_group = os.environ.get("BASE_NAME") + "-AML-RG" subscription_id = os.environ.get("SUBSCRIPTION_ID") tenant_id = os.environ.get("TENANT_ID") app_id = os.environ.get("SP_APP_ID") app_secret = os.environ.get("SP_APP_SECRET") sources_directory_train = os.environ.get("SOURCES_DIR_TRAIN") train_script_path = os.environ.get("TRAIN_SCRIPT_PATH") vm_size = os.environ.get("AML_COMPUTE_CLUSTER_CPU_SKU") compute_name = os.environ.get("AML_COMPUTE_CLUSTER_NAME") model_name = os.environ.get("MODEL_NAME") build_id = os.environ.get("BUILD_BUILDID") pipeline_name = os.environ.get("TRAINING_PIPELINE_NAME") data_path = os.environ.get("DATA_PATH_DATASTORE") model_data_path = os.environ.get("MODEL_DATA_PATH_DATASTORE") # Get Azure machine learning workspace aml_workspace = get_workspace(workspace_name, resource_group, subscription_id, tenant_id, app_id, app_secret) print(aml_workspace) # Get Azure machine learning cluster aml_compute = get_compute(aml_workspace, compute_name, vm_size) if aml_compute is not None: print(aml_compute) model_name = PipelineParameter(name="model_name", default_value=model_name) release_id = PipelineParameter(name="release_id", default_value="0") ds = aml_workspace.get_default_datastore() dataref_folder = ds.path(data_path).as_mount() model_dataref = ds.path(model_data_path).as_mount() # NEED those two folders mounted on datastore and env variables specified in variable groups #ds.upload(src_dir='./VOCdevkit', target_path='VOCdevkit', overwrite=True, show_progress=True) #ds.upload(src_dir='./model_data', target_path='VOCmodel_data', overwrite=True, show_progress=True) yoloEstimator = TensorFlow( source_directory=sources_directory_train + '/training', compute_target=aml_compute, entry_script=train_script_path, pip_packages=[ 'keras', 'pillow', 'matplotlib', 'onnxmltools', 'keras2onnx==1.5.1' ], # recent versions of keras2onnx give conversion issues use_gpu=True, framework_version='1.13') train_step = EstimatorStep(name="Train & Convert Model", estimator=yoloEstimator, estimator_entry_script_arguments=[ "--release_id", release_id, "--model_name", model_name, "--data_folder", dataref_folder, "--model_path", model_dataref ], runconfig_pipeline_params=None, inputs=[dataref_folder, model_dataref], compute_target=aml_compute, allow_reuse=False) print("Step Train & Convert created") train_pipeline = Pipeline(workspace=aml_workspace, steps=[train_step]) train_pipeline.validate() published_pipeline = train_pipeline.publish( name=pipeline_name, description="Model training/retraining pipeline", version=build_id) print(f'Published pipeline: {published_pipeline.name}') print(f'for build {published_pipeline.version}')
def main(): e = Env() # Get Azure machine learning workspace aml_workspace = Workspace.get( name=e.workspace_name, subscription_id=e.subscription_id, resource_group=e.resource_group, ) print("get_workspace:") print(aml_workspace) # Get Azure machine learning cluster aml_compute = get_compute(aml_workspace, e.compute_name, e.vm_size) if aml_compute is not None: print("aml_compute:") print(aml_compute) # Create a reusable Azure ML environment environment = get_environment( aml_workspace, e.aml_env_name, conda_dependencies_file=e.aml_env_train_conda_dep_file, create_new=e.rebuild_env, ) # run_config = RunConfiguration() run_config.environment = environment if e.datastore_name: datastore_name = e.datastore_name else: datastore_name = aml_workspace.get_default_datastore().name run_config.environment.environment_variables[ "DATASTORE_NAME"] = datastore_name # NOQA: E501 model_name_param = PipelineParameter( name="model_name", default_value=e.model_name) # NOQA: E501 dataset_version_param = PipelineParameter(name="dataset_version", default_value=e.dataset_version) data_file_path_param = PipelineParameter(name="data_file_path", default_value="none") caller_run_id_param = PipelineParameter(name="caller_run_id", default_value="none") # NOQA: E501 # Get dataset name dataset_name = e.dataset_name # Check to see if dataset exists if dataset_name not in aml_workspace.datasets: # This call creates an example CSV from sklearn sample data. If you # have already bootstrapped your project, you can comment this line # out and use your own CSV. create_sample_data_csv() # Use a CSV to read in the data set. file_name = "safedriver.csv" if not os.path.exists(file_name): raise Exception( 'Could not find CSV dataset at "%s". If you have bootstrapped your project, you will need to provide a CSV.' # NOQA: E501 % file_name) # NOQA: E501 # Upload file to default datastore in workspace datatstore = Datastore.get(aml_workspace, datastore_name) target_path = "training-data/" datatstore.upload_files( files=[file_name], target_path=target_path, overwrite=True, show_progress=False, ) # Register dataset path_on_datastore = os.path.join(target_path, file_name) dataset = Dataset.Tabular.from_delimited_files( path=(datatstore, path_on_datastore)) dataset = dataset.register( workspace=aml_workspace, name=dataset_name, description="safedriver training data", tags={"format": "CSV"}, create_new_version=True, ) # Create a PipelineData to pass data between steps pipeline_data = PipelineData( "pipeline_data", datastore=aml_workspace.get_default_datastore()) train_step = PythonScriptStep( name="Train Model", script_name=e.train_script_path, compute_target=aml_compute, source_directory=e.sources_directory_train, outputs=[pipeline_data], arguments=[ "--model_name", model_name_param, "--step_output", pipeline_data, "--dataset_version", dataset_version_param, "--data_file_path", data_file_path_param, "--caller_run_id", caller_run_id_param, "--dataset_name", dataset_name, ], runconfig=run_config, allow_reuse=True, ) print("Step Train created") evaluate_step = PythonScriptStep( name="Evaluate Model ", script_name=e.evaluate_script_path, compute_target=aml_compute, source_directory=e.sources_directory_train, arguments=[ "--model_name", model_name_param, "--allow_run_cancel", e.allow_run_cancel, ], runconfig=run_config, allow_reuse=False, ) print("Step Evaluate created") register_step = PythonScriptStep( name="Register Model ", script_name=e.register_script_path, compute_target=aml_compute, source_directory=e.sources_directory_train, inputs=[pipeline_data], arguments=[ "--model_name", model_name_param, "--step_input", pipeline_data, ], # NOQA: E501 runconfig=run_config, allow_reuse=False, ) print("Step Register created") # Check run_evaluation flag to include or exclude evaluation step. if (e.run_evaluation).lower() == "true": print("Include evaluation step before register step.") evaluate_step.run_after(train_step) register_step.run_after(evaluate_step) steps = [train_step, evaluate_step, register_step] else: print("Exclude evaluation step and directly run register step.") register_step.run_after(train_step) steps = [train_step, register_step] train_pipeline = Pipeline(workspace=aml_workspace, steps=steps) train_pipeline._set_experiment_name train_pipeline.validate() published_pipeline = train_pipeline.publish( name=e.pipeline_name, description="Model training/retraining pipeline", version=e.build_id, ) print(f"Published pipeline: {published_pipeline.name}") print(f"for build {published_pipeline.version}")
def main(): e = Env() from azureml.core.authentication import InteractiveLoginAuthentication myten=os.environ.get("AZURE_TENANT_ID") interactive_auth = InteractiveLoginAuthentication(tenant_id=os.environ.get("AZURE_TENANT_ID")) subscription=os.environ.get("CSUBSCRIPTION") workspace_name=e.workspace_name resource_group=e.resource_group aml_workspace = Workspace.get( name = workspace_name, subscription_id = subscription, resource_group=resource_group, auth=interactive_auth ) from ml_service.util.attach_compute import get_compute # Get Azure machine learning cluster # If not present then get_compute will create a compute based on environment variables aml_compute = get_compute( aml_workspace, e.compute_name, e.vm_size) if aml_compute is not None: print("aml_compute:") print(aml_compute) print("SDK version: ", azureml.core.VERSION) ## Variable names that can be passed in as parameter values from azureml.pipeline.core.graph import PipelineParameter from azureml.core import Datastore model_name_param = PipelineParameter( name="model_name", default_value=e.model_name) dataset_version_param = PipelineParameter( name="dataset_version", default_value=e.dataset_version) data_file_path_param = PipelineParameter( name="data_file_path", default_value="none") caller_run_id_param = PipelineParameter( name="caller_run_id", default_value="none") #model_path = PipelineParameter( # name="model_path", default_value=e.model_path) if (e.datastore_name): datastore_name = e.datastore_name else: datastore_name = aml_workspace.get_default_datastore().name # Get the datastore whether it is the default or named store datastore = Datastore.get(aml_workspace, datastore_name) dataset_name = e.dataset_name # Create a reusable Azure ML environment from ml_service.util.manage_environment import get_environment from azureml.core import Environment # RUN Configuration ## Must have this process to work with AzureML-SDK 1.0.85 from azureml.core.runconfig import RunConfiguration, DEFAULT_CPU_IMAGE from azureml.core.conda_dependencies import CondaDependencies try: app_env=Environment(name="smartschedule_env") app_env.register(workspace=aml_workspace) except: print("Environment not found") # Create a new runconfig object aml_run_config = RunConfiguration() aml_run_config.environment.environment_variables["DATASTORE_NAME"] = e.datastore_name # NOQA: E501 # Use the aml_compute you created above. aml_run_config.target = aml_compute # Enable Docker aml_run_config.environment.docker.enabled = True # Set Docker base image to the default CPU-based image aml_run_config.environment.docker.base_image = DEFAULT_CPU_IMAGE #aml_run_config.environment.docker.base_image = "mcr.microsoft.com/azureml/base:0.2.1" # Use conda_dependencies.yml to create a conda environment in the Docker image for execution aml_run_config.environment.python.user_managed_dependencies = False app_conda_deps=CondaDependencies.create( conda_packages=['pandas','scikit-learn', 'libgcc','pyodbc', 'sqlalchemy', 'py-xgboost==0.90'], pip_packages=['azureml-sdk[automl,explain,contrib,interpret]==1.4.0', 'xgboost==0.90', 'azureml-dataprep==1.4.6', 'pyarrow', 'azureml-defaults==1.4.0', 'azureml-train-automl-runtime==1.4.0'], pin_sdk_version=False) # Specify CondaDependencies obj, add necessary packages aml_run_config.environment.python.conda_dependencies = app_conda_deps print ("Run configuration created.") from azure.common.credentials import ServicePrincipalCredentials #from azure.keyvault import KeyVaultClient, KeyVaultAuthentication from azure.keyvault.secrets import SecretClient from azure.identity import DefaultAzureCredential import pandas as pd #import sqlalchemy as sql import pyodbc def get_data(sql_string, columns): credentials = None credential = DefaultAzureCredential() secret_client = SecretClient("https://smrtschd-aml-kv.vault.azure.net", credential=credential) secret = secret_client.get_secret("database-connection") #client = KeyVaultClient(KeyVaultAuthentication(auth_callback)) #secret_bundle = client.get_secret("https://smrtschd-aml-kv.vault.azure.net", "database-connection", "") server = 'starlims-sql.database.windows.net' database = 'QM12_DATA_AUTOMATION' username = '******' password = secret.value driver= '{ODBC Driver 17 for SQL Server}' conn = pyodbc.connect('Driver='+driver+';'+ 'Server='+server+';'+ 'Database='+database+';'+ 'PORT=1433;'+ 'UID='+username+';'+ 'PWD='+password+'; MARS_Connection=Yes' ) try: SQL_Query = pd.read_sql_query(sql_string, conn) df = pd.DataFrame(SQL_Query, columns=columns) return df except Exception as e: print(e) raise sql_str = "SELECT " \ " Dept " \ ", Method " \ ", Servgrp " \ ", Runno " \ ", TestNo " \ ", Testcode " \ ", Total_Duration_Min " \ ", Total_Duration_Hr " \ ", Usrnam " \ ", Eqid " \ ", Eqtype " \ "FROM dbo.Draft " \ "order by TESTCODE, RUNNO, dept, method;" columns = ["Dept", "Method", "Servgrp", "Runno", "TestNo", "Testcode", "Total_Duration_Min", "Total_Duration_Hr", "Usrnam", "Eqid","Eqtype"] from azureml.core import Dataset from sklearn.model_selection import train_test_split if (e.train_dataset_name not in aml_workspace.datasets): df = get_data(sql_str, columns) train_df, test_df=train_test_split(df, test_size=0.2) MY_DIR = "data" CHECK_FOLDER = os.path.isdir(MY_DIR) if not CHECK_FOLDER: os.makedirs(MY_DIR) else: print("Folder ", MY_DIR, " is already created") #files = ["data/analyst_tests.csv"] files = ["data/train_data.csv","data/test_data.csv"] def_file_store = Datastore(aml_workspace, "workspacefilestore") dtfrm = df.to_csv(files[0], header=True, index=False) train_dataframe=train_df.to_csv(files[0], header=True, index=False) test_dataframe=test_df.to_csv(files[1], header=True, index=False) datastore.upload_files( files=files, target_path='data/', overwrite=True ) from azureml.data.data_reference import DataReference blob_input_data_test=DataReference( datastore=datastore, data_reference_name="smartschedulertest", path_on_datastore="data/test_data.csv" ) test_data=Dataset.Tabular.from_delimited_files(blob_input_data_test) test_data.register(aml_workspace, e.test_dataset_name, create_new_version=True) blob_input_data_train=DataReference( datastore=datastore, data_reference_name="smartschedulertrain", path_on_datastore="data/train_data.csv" ) train_data=Dataset.Tabular.from_delimited_files(blob_input_data_train) train_data.register(aml_workspace, e.train_dataset_name, create_new_version=True) else: from azureml.data.data_reference import DataReference print("getting from the datastore instead of uploading") train_data=Dataset.get_by_name(aml_workspace, name=e.train_dataset_name) test_data=Dataset.get_by_name(aml_workspace, name=e.test_dataset_name) # check the training dataset to make sure it has at least 50 records. tdf=train_data.to_pandas_dataframe().head(5) print(tdf.shape) print(tdf) # display the first five rows of the data # create a variable that can be used for other purposes df=train_data.to_pandas_dataframe().head() label_column="Total_Duration_Min" import random import string def randomString(stringLength=15): letters = string.ascii_lowercase return ''.join(random.choice(letters) for i in range(stringLength)) from azureml.core import Experiment experiment = Experiment(aml_workspace, "SmartScheduler_Pipeline") import logging aml_name = 'smart_scheduler_' + randomString(5) print(aml_name) import numpy as np import matplotlib.pyplot as plt from matplotlib.ticker import StrMethodFormatter print(df.head(5)) print(df.shape) print(df.dtypes) #df.hist(column='Dept') list(df.columns.values) # Remove Features that are not necessary. #df.hist(column="Servgrp", bins=4) train_data=train_data.drop_columns(["Runno","TestNo","Total_Duration_Hr"]) test_data=test_data.drop_columns(["Runno","TestNo","Total_Duration_Hr"]) print(train_data.to_pandas_dataframe()) print(test_data.to_pandas_dataframe()) from azureml.automl.core.featurization import FeaturizationConfig # some of the columns could be change to one hot encoding especially if the categorical column featurization_config=FeaturizationConfig() featurization_config.blocked_transformers=['LabelEncoder'] featurization_config.add_column_purpose('Dept', 'CategoricalHash') featurization_config.add_transformer_params('HashOneHotEncoder',['Method'], {"number_of_bits":3}) featurization_config.add_column_purpose('Servgrp', 'CategoricalHash') featurization_config.add_column_purpose('Testcode', 'Numeric') featurization_config.add_column_purpose('Usrnam', 'CategoricalHash') featurization_config.add_column_purpose('Eqid', 'CategoricalHash') featurization_config.add_column_purpose('Eqtype', 'CategoricalHash') from azureml.pipeline.core import Pipeline, PipelineData from azureml.pipeline.steps import PythonScriptStep #train_model_folder = './scripts/trainmodel' automl_settings = { "iteration_timeout_minutes": 5, "iterations": 5, "enable_early_stopping": True, "primary_metric": 'spearman_correlation', "verbosity": logging.INFO, "n_cross_validation":5 } automl_config = AutoMLConfig(task="regression", debug_log='automated_ml_errors.log', #path = train_model_folder, training_data=train_data, featurization=featurization_config, blacklist_models=['XGBoostRegressor'], label_column_name=label_column, compute_target=aml_compute, **automl_settings) from azureml.pipeline.steps import AutoMLStep from azureml.pipeline.core import TrainingOutput metrics_output_name = 'metrics_output' best_model_output_name='best_model_output' metrics_data = PipelineData(name = 'metrics_data', datastore = datastore, pipeline_output_name=metrics_output_name, training_output=TrainingOutput(type='Metrics')) model_data = PipelineData(name='model_data', datastore=datastore, pipeline_output_name=best_model_output_name, training_output=TrainingOutput(type='Model')) trainWithAutomlStep = AutoMLStep( name=aml_name, automl_config=automl_config, passthru_automl_config=False, outputs=[metrics_data, model_data], allow_reuse=True ) evaluate_step = PythonScriptStep( name="Evaluate Model", script_name='./evaluate/evaluate_model.py', # e.evaluate_script_path, compute_target=aml_compute, source_directory='../app', arguments=[ "--model_name", model_name_param, "--allow_run_cancel", e.allow_run_cancel ] ) register_step = PythonScriptStep( name="Register Model ", script_name='register/register_model2.py', #e.register_script_path, compute_target=aml_compute, source_directory='../app', inputs=[model_data], arguments=[ "--model_name", model_name_param, "--model_path", model_data, "--ds_name", e.train_dataset_name ], runconfig=aml_run_config, allow_reuse=False ) if ((e.run_evaluation).lower() == 'true'): print("Include evaluation step before register step.") evaluate_step.run_after(trainWithAutomlStep) register_step.run_after(evaluate_step) pipeline_steps = [ trainWithAutomlStep, evaluate_step, register_step ] else: print("Exclude the evaluation step and run register step") register_step.run_after(trainWithAutomlStep) pipeline_steps = [ trainWithAutomlStep, register_step ] print( "this is the value for execute pipeline: {}".format(e.execute_pipeline)) if( (e.execute_pipeline).lower() =='true' ): # Execute the pipe normally during testing and debugging print("Pipeline submitted for execution.") pipeline = Pipeline(workspace = aml_workspace, steps=pipeline_steps) pipeline_run = experiment.submit(pipeline) pipeline_run.wait_for_completion() print("Pipeline is built.") else: # Generates pipeline that will be called in ML Ops train_pipeline = Pipeline(workspace=aml_workspace, steps=pipeline_steps) train_pipeline._set_experiment_name train_pipeline.validate() published_pipeline = train_pipeline.publish( name=e.pipeline_name, description="Model training/retraining pipeline", version=e.build_id ) print(f'Published pipeline: {published_pipeline.name}') print(f'for build {published_pipeline.version}')