def client_from(configuration): client = training.BatchAIManagementClient( credentials=ServicePrincipalCredentials( client_id=configuration.client_id, secret=configuration.secret, token_uri=configuration.token), subscription_id=configuration.subscription_id) return client
def create_batchai_client(configuration): client = training.BatchAIManagementClient( credentials=ServicePrincipalCredentials( client_id=configuration.aad_client_id, secret=configuration.aad_secret_key, tenant=configuration.aad_tenant), subscription_id=configuration.subscription_id, base_url=configuration.url) return client
def get_client(config): ''' Connect to Batch AI ''' client = training.BatchAIManagementClient( credentials=ServicePrincipalCredentials( client_id=config.bait_aad_client_id, secret=config.bait_aad_secret, token_uri=config.bait_aad_token_uri), subscription_id=config.bait_subscription_id, base_url=None) return (client)
def create_batch_ai_client_and_resource(config): # Create Batch AI client creds = ServicePrincipalCredentials(client_id=config.aad_client_id, secret=config.aad_secret_key, tenant=config.aad_tenant) batchai_client = batchai.BatchAIManagementClient(credentials=creds, subscription_id=config.subscription_id) # Create a resource group resource_management_client = ResourceManagementClient( credentials=creds, subscription_id=config.subscription_id) resource = resource_management_client.resource_groups.create_or_update( config.workspace_resource_group, {'location': config.location}) return batchai_client, resource
configuration_file_name = "configuration.json" with open(configuration_file_name, 'r') as f: cfg = json.load(f) # load utilities of Batch AI sys.path.append(cfg['utils_dir']) import utilities as utils from utilities.job_factory import ParameterSweep, NumericParameter, DiscreteParameter # connect to Batch AI workspace and cluster creds = ServicePrincipalCredentials( client_id=cfg['active_directory']['client_id'], secret=cfg['active_directory']['client_secret'], tenant=cfg['active_directory']['tenant_id']) batchai_client = batchai.BatchAIManagementClient( credentials=creds, subscription_id=cfg['subscription_id']) cluster = batchai_client.clusters.get(cfg['resource_group'], cfg['batch_ai']['workspace'], cfg['batch_ai']['cluster_name']) # define grid of tuned hyperparameters param_specs = [ DiscreteParameter(parameter_name="LATENT_DIM", values=[5, 10, 15]), DiscreteParameter(parameter_name="HIDDEN_LAYERS", values=[1, 2, 3]), DiscreteParameter(parameter_name="BATCH_SIZE", values=[8, 16, 32]), DiscreteParameter(parameter_name="T", values=[72, 168, 336]), DiscreteParameter(parameter_name="LEARNING_RATE", values=[0.01, 0.001, 0.0001]), DiscreteParameter(parameter_name="ALPHA", values=[0.1, 0.001, 0]) ]
experiment_name = "<EXPERIMENT NAME>" node_count = 1 std_out_err_path_prefix = '<FILE SHARE TO SAVE ERROR LOGS>' # DEVICE TYPES device_ids = ["Transformer001", "Transformer003", "Transformer005", "Transformer007", "Transformer009"] # LOG IN AND CREATE CLIENT credentials = ServicePrincipalCredentials( client_id=CLIENT, secret=KEY, tenant=TENANT_ID ) batchai_client = batchai.BatchAIManagementClient( credentials=credentials, subscription_id=subscription_id) # GET CLUSTER OBJECT cluster = batchai_client.clusters.get( resource_group_name, workspace_name, cluster_name) # DELETE JOBS IF EXISTS for j in batchai_client.jobs.list_by_experiment(resource_group_name, workspace_name, experiment_name): print("Deleting job- ",j.name) batchai_client.jobs.delete( resource_group_name, workspace_name, experiment_name, j.name) print ("Deleted job- {0}".format(j.name)) # RUN AN ASYNC JOB FOR EACH DEVICE TYPE (EXECUTE TRAIN.PY FOR EACH DEVICE TYPE) for device_id in device_ids: job_name = 'train-{0}'.format(device_id)
def setup_bai( aad_client_id: str = None, aad_secret: str = None, aad_tenant: str = None, subscription_id: str = None, rg: str = None, location: str = None, ) -> 'batchai.BatchAIManagementClient': ''' Setup credentials, batch AI client, and the resource group that the resources will be created in All optional arguments will default to using the associated environment variable if the parameter is not provided. Args: aad_client_id (str, optional): The client id you get from creating your Service Principle. aad_secret (str, optional): The secret key you get from creating your Service Principle. aad_tenant (str, optional): The tenant id that your Service Principle is created in. subscription_id (str, optional): The subscription id you wish for your Batch AI resources to be created in. rg (str, optional): The Resource Group you will create your work in. location (str, optional): The location/region that will create your Azure resources in. Returns: BatchAIManagementClient: An instance of the Batch AI managment client that can be used to manage Batch AI resources. ''' aad_client_id = aad_client_id or os.getenv('AAD_CLIENT_ID') aad_tenant = aad_tenant or os.getenv('AAD_TENANT') aad_secret = aad_secret or os.getenv('AAD_SECRET') subscription_id = subscription_id or os.getenv('SUBSCRIPTION_ID') rg = rg or os.getenv('RESOURCE_GROUP') location = location or os.getenv('REGION') assert aad_client_id assert aad_tenant assert aad_secret assert subscription_id assert rg assert location creds = ServicePrincipalCredentials( client_id=aad_client_id, secret=aad_secret, tenant=aad_tenant ) resource_management_client = ResourceManagementClient( credentials=creds, subscription_id=subscription_id ) resource = resource_management_client \ .resource_groups.create_or_update(rg, { 'location': location }) batchai_client = batchai.BatchAIManagementClient( credentials=creds, subscription_id=subscription_id ) return batchai_client
import zipfile from azure.storage.file import FileService from azure.storage.blob import BlockBlobService import azure.mgmt.batchai.models as models import azure.mgmt.batchai as batchai from azure.common.credentials import ServicePrincipalCredentials tenant = "<Insert Correct Value Here>" subscription = "<Insert Correct Value Here>" resource_group_name = "batchai" credentials = ServicePrincipalCredentials(client_id="<Insert Correct Value Here>", secret="<Insert Correct Value Here>", token_uri="https://login.microsoftonline.com/{0}/oauth2/token".format(tenant)) client = batchai.BatchAIManagementClient( credentials=credentials, subscription_id=subscription, base_url="") from azure.mgmt.resource import ResourceManagementClient resource_management_client = ResourceManagementClient(credentials=credentials, subscription_id=subscription) group = resource_management_client.resource_groups.create_or_update( resource_group_name, {'location': 'northeurope'}) from azure.storage.file import FileService storage_account_name = "<Insert Correct Value Here>" storage_account_key = "<Insert Correct Value Here>" fileshare = "data"