def upload_files_to_azure(cfg, ws): ''' look in the cfg object to file directories and files to upload to AFS and ABS input params : ws : Description : aml workspace object ws : Type : aml workspace object (defined in azureml.core.workspace.Workspace) ''' for ref in cfg.DataReference.localDirectoryBlobList: uploadContentBeforeRun = ref.uploadContentBeforeRun if uploadContentBeforeRun: overwriteOnUpload = ref.overwriteOnUpload remoteBlobContainer = ref.remoteBlobContainer localDirectoryName = ref.localDirectoryName remoteMountPath = ref.remoteMountPath ds = Datastore(workspace = ws, name = remoteBlobContainer) ds.upload(src_dir=localDirectoryName, target_path=remoteMountPath, overwrite=overwriteOnUpload, show_progress=True) for ref in cfg.DataReference.localDirectoryFilesList: uploadContentBeforeRun = ref.uploadContentBeforeRun if uploadContentBeforeRun: overwriteOnUpload = ref.overwriteOnUpload remoteFileShare = ref.remoteFileShare localDirectoryName = ref.localDirectoryName remoteMountPath = ref.remoteMountPath ds = Datastore(workspace = ws, name = remoteFileShare) ds.upload(src_dir = localDirectoryName, target_path=remoteMountPath, overwrite=overwriteOnUpload, show_progress=True)
def create_and_attach_file_storage(cfg, ws): if len(cfg.DataReference.localDirectoryFilesList) > 0: for ref in cfg.DataReference.localDirectoryFilesList: log.info("Attempting to create file share '%s' on storage account '%s'.", ref.remoteFileShare, ref.storageAccountName) file_service = FileService(ref.storageAccountName, ref.storageAccountKey) exist = file_service.create_share(ref.remoteFileShare, fail_on_exist=False) if exist: log.info("File Share '%s' on storage account '%s' created.", ref.remoteFileShare, ref.storageAccountName) else: log.info("File Share '%s' on storage account '%s' already existed.", ref.remoteFileShare, ref.storageAccountName) # Get most recent list of datastores linked to current workspace datastores = ws.datastores() # Validate if share_ds is created ds = None if ref.dataref_id not in datastores else Datastore(workspace = ws, name = ref.dataref_id) # Register the DS to the workspace if ds: if ds.account_name == ref.storageAccountName and ds.container_name == ref.remoteFileShare: recreate = False else: recreate = True # also remove the existing reference ds.unregister() else: recreate = True if recreate: log.info('Registering file share "{}" to AML datastore for AML workspace "{}" under datastore id "{}".'.format(ref.remoteFileShare, ws.name, ref.dataref_id)) ds = Datastore.register_azure_file_share(workspace = ws, datastore_name = ref.dataref_id, file_share_name = ref.remoteFileShare, account_name = ref.storageAccountName, account_key= ref.storageAccountKey, overwrite=True, ) else: log.info('File share "{}" under AML workspace "{}" already registered under datastore id "{}".'.format(ref.remoteFileShare, ws.name, ref.dataref_id))
def create_dataset(ws): kaggle_api.dataset_download_file('divg07/malware-analysis-dataset','data.csv') data = pd.read_csv( './data.csv.zip', compression='zip', sep='|' ) # Clean dataset data = clean_data(data) # Register Dataset in Workspace datastore = Datastore(ws) name = "Malware Dataset" description_text = "Malware DataSet for Udacity Capstone Project" dataset = TabularDatasetFactory.register_pandas_dataframe(data, datastore, name, description=description_text) return dataset
def create_and_attach_blob_storage(cfg, ws): """ If required, creates the blob storage containers in the datareferences of cfg """ if len(cfg.DataReference.localDirectoryBlobList) > 0: for ref in cfg.DataReference.localDirectoryBlobList: log.info("Attempting to create Blob Container '%s' on storage account '%s'.", ref.remoteBlobContainer, ref.storageAccountName) blob_service = BlockBlobService(ref.storageAccountName, ref.storageAccountKey) exist = blob_service.create_container(ref.remoteBlobContainer, fail_on_exist=False) if exist: log.info("Blob Container '%s' on storage account '%s' created.", ref.remoteBlobContainer, ref.storageAccountName) else: log.info("Blob Container '%s' on storage account '%s' already existed.", ref.remoteBlobContainer, ref.storageAccountName) # Get most recent list of datastores linked to current workspace datastores = ws.datastores() # Validate if blob_ds is created ds = None if ref.dataref_id not in datastores else Datastore(workspace = ws, name = ref.dataref_id) # If DS exists and isn't mapped to the right place if ds: if ds.account_name == ref.storageAccountName and ds.container_name == ref.remoteBlobContainer: recreate = False else: recreate = True # also remove the existing reference ds.unregister() else: recreate = True if recreate: log.info('Registering blob "{}" to AML datastore for AML workspace "{}" under datastore id "{}".'.format(ref.remoteBlobContainer, ws.name, ref.dataref_id)) ds = Datastore.register_azure_blob_container(workspace = ws, datastore_name = ref.dataref_id, container_name = ref.remoteBlobContainer, account_name = ref.storageAccountName, account_key = ref.storageAccountKey, overwrite = True, # Overwrites the datastore (not the data itself, the object) if it already is part of this workspace ) else: log.info('Blob "{}" under AML workspace "{}" already registered under datastore id "{}".'.format(ref.remoteBlobContainer, ws.name, ref.dataref_id))
def main(): """ Builds the Azure ML pipeline for data engineering and model training. """ databricks_workspace_name = os.environ['DATABRICKS_WORKSPACE_NAME'] training_data_account_name = os.environ['TRAINING_DATA_ACCOUNT_NAME'] build_id = os.getenv('BUILD_BUILDID', 0) # Get Azure machine learning workspace aml_workspace = Workspace.get( name=os.environ['AML_WORKSPACE_NAME'], subscription_id=os.environ['SUBSCRIPTION_ID'], resource_group=os.environ['RESOURCE_GROUP'], ) print(aml_workspace) # Generate Databricks credentials, see https://aka.ms/databricks-aad dbricks_region = aml_workspace.location dbricks_api = f"https://{dbricks_region}.azuredatabricks.net/api/2.0" dbricks_client = databricks_client.create(dbricks_api) dbricks_client.auth_azuread(resource_group=aml_workspace.resource_group, workspace_name=databricks_workspace_name) dbricks_client.ensure_available() # Attach Databricks as Azure ML training compute dbricks_compute_name = "databricks" dbricks_compute = get_databricks_compute( aml_workspace, dbricks_compute_name, ) if dbricks_compute is None: pat_token = dbricks_client.post( 'token/create', json={"comment": "Azure ML Token generated by Build " + build_id })['token_value'] dbricks_compute = create_databricks_compute( aml_workspace, databricks_workspace_name, dbricks_compute_name, pat_token, ) print("dbricks_compute:") print(dbricks_compute) # Create Databricks instance pool pool_name = "azureml_training" instance_pool_id = get_instance_pool(dbricks_client, pool_name) if not instance_pool_id: dbricks_client.post('instance-pools/create', json={ "instance_pool_name": pool_name, "node_type_id": "Standard_D3_v2", "idle_instance_autotermination_minutes": 10, "preloaded_spark_versions": [DATABRICKS_RUNTIME_VERSION], }) instance_pool_id = get_instance_pool(dbricks_client, pool_name) notebook_folder = f"/Shared/AzureMLDeployed" workspace_datastore = Datastore(aml_workspace, "workspaceblobstore") # Create a datastore for the training data container credentials, subscription = get_azure_cli_credentials() storage_client = StorageManagementClient(credentials, subscription) training_storage_keys = storage_client.storage_accounts.list_keys( aml_workspace.resource_group, training_data_account_name) training_datastore = Datastore.register_azure_blob_container( workspace=aml_workspace, datastore_name="trainingdata", container_name="trainingdata", account_name=training_data_account_name, account_key=training_storage_keys.keys[0].value, ) # FEATURE ENGINEERING STEP (DATABRICKS) # Create feature engineering pipeline step training_data_input = DataReference(datastore=training_datastore, path_on_datastore="/", data_reference_name="training") feature_eng_output = PipelineData("feature_engineered", datastore=workspace_datastore) notebook_path = upload_notebook(dbricks_client, notebook_folder, "code/prepare", "feature_engineering") training_dataprep_step = DatabricksStep( name="FeatureEngineering", inputs=[training_data_input], outputs=[feature_eng_output], spark_version=DATABRICKS_RUNTIME_VERSION, instance_pool_id=instance_pool_id, num_workers=3, notebook_path=notebook_path, run_name="FeatureEngineering", compute_target=dbricks_compute, allow_reuse=True, ) # You can add Azure ML model training tasks using # feature_eng_output as input. # ... # Create Azure ML Pipeline steps = [training_dataprep_step] ml_pipeline = Pipeline(workspace=aml_workspace, steps=steps) ml_pipeline.validate() published_pipeline = ml_pipeline.publish( name="Feature Engineering", description="Feature engineering pipeline", version=build_id, ) print(f"Published pipeline: {published_pipeline.name}") print(f"for build {published_pipeline.version}") # When running in Azure DevOps, set AMLPIPELINE_ID variable # for AML Pipeline task in next job print("Setting Azure DevOps variable") print(f"##vso[task.setvariable variable=AMLPIPELINE_ID;isOutput=true]" f"{published_pipeline.id}")
from azureml.pipeline.core import Schedule from azureml.core.datastore import Datastore datastore = Datastore(workspace="Playground", name="workspaceblobstore") schedule = Schedule.create(workspace, name="TestSchedule", pipeline_id="3100e87c-3300-400b-a5a5-470e85a100b3" experiment_name="working version", datastore=datastore, polling_interval=25, path_on_datastore="file/path")
def InitAML(model_name, env, svcpw, interactive=False, create_ws=False): print("Environment is ", env) configFilePath = "./environment_setup/Config/config_" + env + ".ini" configFile = ConfigParser() configFile.read(configFilePath) svc_pr_pd = svcpw tenant_id = configFile.get('PARAMS', 'tenant_id') service_principal_id = configFile.get('PARAMS', 'service_principal_id') subscription_id = configFile.get('PARAMS', 'subscription_id') resource_group = configFile.get('PARAMS', 'resource_group') blobname = configFile.get('PARAMS', 'BlobName') workspace_name = configFile.get('PARAMS', 'WorkSpace') data_factory_name = configFile.get('PARAMS', 'Data_factory_name') location = configFile.get('PARAMS', 'location') fp = './' + model_name + '/aml_service/setup.ini' conf = ConfigParser() conf.read(fp) AML_COMPUTE_CLUSTER_NAME = conf.get('PARAMS', 'AML_COMPUTE_CLUSTER_NAME') AML_COMPUTE_CLUSTER_MIN_NODES = conf.get('PARAMS', 'AML_COMPUTE_CLUSTER_MIN_NODES') AML_COMPUTE_CLUSTER_MAX_NODES = conf.get('PARAMS', 'AML_COMPUTE_CLUSTER_MAX_NODES') AML_COMPUTE_CLUSTER_SKU = conf.get('PARAMS', 'AML_COMPUTE_CLUSTER_SKU') if interactive: auth = InteractiveLoginAuthentication(tenant_id=tenant_id) else: auth = ServicePrincipalAuthentication( tenant_id=tenant_id, service_principal_id=service_principal_id, service_principal_password=svc_pr_pd) subscription_id = subscription_id resource_group = resource_group try: ws = Workspace(subscription_id=subscription_id, resource_group=resource_group, workspace_name=workspace_name, auth=auth) print('Library configuration succeeded') except: if create_ws: ws = Workspace.create(name=workspace_name, auth=auth, subscription_id=subscription_id, resource_group=resource_group, create_resource_group=False, location=location) print('Workspace not found and is created') else: print('Workspace not found and not created') print('workspace_name:', ws.name, '\nworkspace_location:', ws.location, '\nworkspace_resource_group:', ws.resource_group, sep='\t') # choose a name for your cluster compute_name = os.environ.get("AML_COMPUTE_CLUSTER_NAME", AML_COMPUTE_CLUSTER_NAME) if compute_name in ws.compute_targets: compute_target = ws.compute_targets[compute_name] if compute_target and type(compute_target) is AmlCompute: print('found compute target. just use it. ' + compute_name) else: print('creating a new compute target...') compute_min_nodes = os.environ.get("AML_COMPUTE_CLUSTER_MIN_NODES", AML_COMPUTE_CLUSTER_MIN_NODES) compute_max_nodes = os.environ.get("AML_COMPUTE_CLUSTER_MAX_NODES", AML_COMPUTE_CLUSTER_MAX_NODES) # This example uses CPU VM. For using GPU VM, set SKU to STANDARD_NC6 vm_size = os.environ.get("AML_COMPUTE_CLUSTER_SKU", AML_COMPUTE_CLUSTER_SKU) provisioning_config = AmlCompute.provisioning_configuration( vm_size=vm_size, min_nodes=compute_min_nodes, max_nodes=compute_max_nodes) # create the cluster compute_target = ComputeTarget.create(ws, compute_name, provisioning_config) # can poll for a minimum number of nodes and for a specific timeout. # if no min node count is provided it will use the scale settings for the cluster compute_target.wait_for_completion(show_output=True, min_node_count=None, timeout_in_minutes=20) # For a more detailed view of current AmlCompute status, use get_status() print(compute_target.get_status().serialize()) try: datastore = Datastore(ws, name=blobname) print("Found Blob Datastore with name: %s" % datastore) except: print("No datastore with name: %s" % blobname) sys.exit(-1) try: data_factory = DataFactoryCompute(ws, data_factory_name) print('data_factory ', data_factory) except ComputeTargetException as e: if 'ComputeTargetNotFound' in e.message: print('Data factory Compute not found, creating...') provisioning_config = DataFactoryCompute.provisioning_configuration( ) data_factory = ComputeTarget.create(ws, data_factory_name, provisioning_config) data_factory.wait_for_completion() else: print('Data factory Compute not found, Entering Else Section...') raise e return datastore, compute_target, ws, data_factory
workspace_name="<your workspace name>" if subscription_id.startswith("<"): raise ex else: # write and reload from config file config = {"Scope": "/subscriptions/" + subscription_id + "/resourceGroups/" + resource_group + "/providers/Microsoft.MachineLearningServices/workspaces/" + workspace_name +"/projects/samples"} import json import os os.makedirs(os.path.dirname(config_path), exist_ok=True) with open(config_path, "w") as fo: fo.write(json.dumps(config)) ws = Workspace.from_config(path=config_path) from azureml.core.datastore import Datastore relevance_datastore = Datastore(ws, 'adls_relevance09') # Retrieve or create the computer target from azureml.core.compute import AmlCompute, ComputeTarget from azureml.core.compute_target import ComputeTargetException cluster_name = "Cmpt-112GB-16Cr" # cluster_name = "Cmpt-512GB-64Cr" if cluster_name not in ws.compute_targets: print('Creating a new compute target...') compute_config = AmlCompute.provisioning_configuration(vm_size="Standard_D2_v2", max_nodes=4) compute_target = ComputeTarget.create(ws, cluster_name, compute_config) compute_target.wait_for_completion(show_output=True, timeout_in_minutes=20)
ws = Workspace.get(name=workspace_name, subscription_id=subscription_id, resource_group=resource_group, auth=service_principal) return (ws) if __name__ == '__main__': global run run = Run.get_context() auth_params = get_args() ws = get_ws(auth_params) datastore_names = list(ws.datastores.keys()) def_data_store = ws.get_default_datastore() def_blob_store = Datastore(ws, "workspaceblobstore") data_temp_folder = os.path.join(cwd, "data_temp") create_folders([data_temp_folder]) dataset = { 'dataset': "https://github.com/rouzbeh-afrasiabi/PublicDatasets/raw/master/train.csv.zip" } word_vectors = { "en_vectors_web_lg": "https://github.com/explosion/spacy-models/releases/download/en_vectors_web_lg-2.1.0/en_vectors_web_lg-2.1.0.tar.gz" } toDownload = [dataset, word_vectors] download_files(toDownload, data_temp_folder)
def create_DDoS_datasets(ws): dtypes = { 'Src IP': 'category', 'Src Port': 'uint16', 'Dst IP': 'category', 'Dst Port': 'uint16', 'Protocol': 'category', 'Flow Duration': 'uint32', 'Tot Fwd Pkts': 'uint32', 'Tot Bwd Pkts': 'uint32', 'TotLen Fwd Pkts': 'float32', 'TotLen Bwd Pkts': 'float32', 'Fwd Pkt Len Max': 'float32', 'Fwd Pkt Len Min': 'float32', 'Fwd Pkt Len Mean': 'float32', 'Fwd Pkt Len Std': 'float32', 'Bwd Pkt Len Max': 'float32', 'Bwd Pkt Len Min': 'float32', 'Bwd Pkt Len Mean': 'float32', 'Bwd Pkt Len Std': 'float32', 'Flow Byts/s': 'float32', 'Flow Pkts/s': 'float32', 'Flow IAT Mean': 'float32', 'Flow IAT Std': 'float32', 'Flow IAT Max': 'float32', 'Flow IAT Min': 'float32', 'Fwd IAT Tot': 'float32', 'Fwd IAT Mean': 'float32', 'Fwd IAT Std': 'float32', 'Fwd IAT Max': 'float32', 'Fwd IAT Min': 'float32', 'Bwd IAT Tot': 'float32', 'Bwd IAT Mean': 'float32', 'Bwd IAT Std': 'float32', 'Bwd IAT Max': 'float32', 'Bwd IAT Min': 'float32', 'Fwd PSH Flags': 'category', 'Bwd PSH Flags': 'category', 'Fwd URG Flags': 'category', 'Bwd URG Flags': 'category', 'Fwd Header Len': 'uint32', 'Bwd Header Len': 'uint32', 'Fwd Pkts/s': 'float32', 'Bwd Pkts/s': 'float32', 'Pkt Len Min': 'float32', 'Pkt Len Max': 'float32', 'Pkt Len Mean': 'float32', 'Pkt Len Std': 'float32', 'Pkt Len Var': 'float32', 'FIN Flag Cnt': 'category', 'SYN Flag Cnt': 'category', 'RST Flag Cnt': 'category', 'PSH Flag Cnt': 'category', 'ACK Flag Cnt': 'category', 'URG Flag Cnt': 'category', 'CWE Flag Count': 'category', 'ECE Flag Cnt': 'category', 'Down/Up Ratio': 'float32', 'Pkt Size Avg': 'float32', 'Fwd Seg Size Avg': 'float32', 'Bwd Seg Size Avg': 'float32', 'Fwd Byts/b Avg': 'uint32', 'Fwd Pkts/b Avg': 'uint32', 'Fwd Blk Rate Avg': 'uint32', 'Bwd Byts/b Avg': 'uint32', 'Bwd Pkts/b Avg': 'uint32', 'Bwd Blk Rate Avg': 'uint32', 'Subflow Fwd Pkts': 'uint32', 'Subflow Fwd Byts': 'uint32', 'Subflow Bwd Pkts': 'uint32', 'Subflow Bwd Byts': 'uint32', 'Init Fwd Win Byts': 'uint32', 'Init Bwd Win Byts': 'uint32', 'Fwd Act Data Pkts': 'uint32', 'Fwd Seg Size Min': 'uint32', 'Active Mean': 'float32', 'Active Std': 'float32', 'Active Max': 'float32', 'Active Min': 'float32', 'Idle Mean': 'float32', 'Idle Std': 'float32', 'Idle Max': 'float32', 'Idle Min': 'float32', 'Label': 'category' } data = pd.read_csv( './final_dataset.csv', parse_dates=['Timestamp'], usecols=[*dtypes.keys(), 'Timestamp'], engine='c', low_memory=True, na_values=np.inf ) # There are over 12 million rows in this orignal dataset. For this project, that much data is taking far too long, so I'm randomly sampling only .5% of the data data = data.sample(frac=0.005) # Register Base Dataset in Workspace datastore = Datastore(ws) name = "DDoS Dataset" description_text = "DDoS DataSet for Udacity Capstone Project" dataset = TabularDatasetFactory.register_pandas_dataframe(data, datastore, name, description=description_text) # Clean dataset and register the clean version cleaned_data = clean_data(data) clean_dataset_name = "Clean DDoS Dataset" clean_description_text = description_text + " that has been cleaned" clean_dataset = TabularDatasetFactory.register_pandas_dataframe(cleaned_data, datastore, clean_dataset_name, description=clean_description_text)
cv_results['n_features'] = X.shape[1] cv_results['y_0'] = y.tolist().count(0) cv_results['y_1'] = y.tolist().count(1) print(cv_results["mean_test_pr_auc"].to_string(index=False)) run.log(name="mean_test_pr_auc", value=cv_results["mean_test_pr_auc"].to_string(index=False)) if not os.path.isdir(args.train_model): os.makedirs(args.train_model, exist_ok=True) timestamp_id = datetime.datetime.now() time = timestamp_id.strftime("%m-%d-%Y_%H%M") model_name = "{}__{}.json".format(args.repo_owner, args.repo_name) output_path = os.path.join(args.train_model, model_name) with open(output_path, 'w') as outfile: cv_results.to_json(outfile, orient='table', index=False) # Get the blob storage associated with the workspace pipeline_datastore = Datastore(ws, "datastore_pipeline") #Upload production model to main blob folder pipeline_datastore.upload_files([args.train_model + '/' + model_name], target_path="train_model" + '/' + args.repo_owner + '/' + args.repo_name + '/' + time, overwrite=True) print("Model is trained!")