def client_from(configuration):
client = training.BatchAIManagementClient(
credentials=ServicePrincipalCredentials(
client_id=configuration.client_id,
secret=configuration.secret,
token_uri=configuration.token),
subscription_id=configuration.subscription_id)
return client
def create_batchai_client(configuration):
client = training.BatchAIManagementClient(
credentials=ServicePrincipalCredentials(
client_id=configuration.aad_client_id,
secret=configuration.aad_secret_key,
tenant=configuration.aad_tenant),
subscription_id=configuration.subscription_id,
base_url=configuration.url)
return client
def get_client(config):
''' Connect to Batch AI '''
client = training.BatchAIManagementClient(
credentials=ServicePrincipalCredentials(
client_id=config.bait_aad_client_id,
secret=config.bait_aad_secret,
token_uri=config.bait_aad_token_uri),
subscription_id=config.bait_subscription_id,
base_url=None)
return (client)
def create_batch_ai_client_and_resource(config):
# Create Batch AI client
creds = ServicePrincipalCredentials(client_id=config.aad_client_id, secret=config.aad_secret_key, tenant=config.aad_tenant)
batchai_client = batchai.BatchAIManagementClient(credentials=creds, subscription_id=config.subscription_id)
# Create a resource group
resource_management_client = ResourceManagementClient(
credentials=creds, subscription_id=config.subscription_id)
resource = resource_management_client.resource_groups.create_or_update(
config.workspace_resource_group, {'location': config.location})
return batchai_client, resource
configuration_file_name = "configuration.json"
with open(configuration_file_name, 'r') as f:
cfg = json.load(f)
# load utilities of Batch AI
sys.path.append(cfg['utils_dir'])
import utilities as utils
from utilities.job_factory import ParameterSweep, NumericParameter, DiscreteParameter
# connect to Batch AI workspace and cluster
creds = ServicePrincipalCredentials(
client_id=cfg['active_directory']['client_id'],
secret=cfg['active_directory']['client_secret'],
tenant=cfg['active_directory']['tenant_id'])
batchai_client = batchai.BatchAIManagementClient(
credentials=creds, subscription_id=cfg['subscription_id'])
cluster = batchai_client.clusters.get(cfg['resource_group'],
cfg['batch_ai']['workspace'],
cfg['batch_ai']['cluster_name'])
# define grid of tuned hyperparameters
param_specs = [
DiscreteParameter(parameter_name="LATENT_DIM", values=[5, 10, 15]),
DiscreteParameter(parameter_name="HIDDEN_LAYERS", values=[1, 2, 3]),
DiscreteParameter(parameter_name="BATCH_SIZE", values=[8, 16, 32]),
DiscreteParameter(parameter_name="T", values=[72, 168, 336]),
DiscreteParameter(parameter_name="LEARNING_RATE",
values=[0.01, 0.001, 0.0001]),
DiscreteParameter(parameter_name="ALPHA", values=[0.1, 0.001, 0])
]
experiment_name = "<EXPERIMENT NAME>"
node_count = 1
std_out_err_path_prefix = '<FILE SHARE TO SAVE ERROR LOGS>'
# DEVICE TYPES
device_ids = ["Transformer001", "Transformer003",
"Transformer005", "Transformer007", "Transformer009"]
# LOG IN AND CREATE CLIENT
credentials = ServicePrincipalCredentials(
client_id=CLIENT,
secret=KEY,
tenant=TENANT_ID
)
batchai_client = batchai.BatchAIManagementClient(
credentials=credentials, subscription_id=subscription_id)
# GET CLUSTER OBJECT
cluster = batchai_client.clusters.get(
resource_group_name, workspace_name, cluster_name)
# DELETE JOBS IF EXISTS
for j in batchai_client.jobs.list_by_experiment(resource_group_name, workspace_name, experiment_name):
print("Deleting job- ",j.name)
batchai_client.jobs.delete(
resource_group_name, workspace_name, experiment_name, j.name)
print ("Deleted job- {0}".format(j.name))
# RUN AN ASYNC JOB FOR EACH DEVICE TYPE (EXECUTE TRAIN.PY FOR EACH DEVICE TYPE)
for device_id in device_ids:
job_name = 'train-{0}'.format(device_id)
def setup_bai(
aad_client_id: str = None,
aad_secret: str = None,
aad_tenant: str = None,
subscription_id: str = None,
rg: str = None,
location: str = None,
) -> 'batchai.BatchAIManagementClient':
'''
Setup credentials, batch AI client, and the resource
group that the resources will be created in
All optional arguments will default to using the
associated environment variable if the parameter
is not provided.
Args:
aad_client_id (str, optional): The client id you get
from creating your Service Principle.
aad_secret (str, optional): The secret key you get
from creating your Service Principle.
aad_tenant (str, optional): The tenant id that your
Service Principle is created in.
subscription_id (str, optional): The subscription id
you wish for your Batch AI resources to be created
in.
rg (str, optional): The Resource Group you will
create your work in.
location (str, optional): The location/region that
will create your Azure resources in.
Returns:
BatchAIManagementClient: An instance of the Batch AI
managment client that can be used to manage Batch
AI resources.
'''
aad_client_id = aad_client_id or os.getenv('AAD_CLIENT_ID')
aad_tenant = aad_tenant or os.getenv('AAD_TENANT')
aad_secret = aad_secret or os.getenv('AAD_SECRET')
subscription_id = subscription_id or os.getenv('SUBSCRIPTION_ID')
rg = rg or os.getenv('RESOURCE_GROUP')
location = location or os.getenv('REGION')
assert aad_client_id
assert aad_tenant
assert aad_secret
assert subscription_id
assert rg
assert location
creds = ServicePrincipalCredentials(
client_id=aad_client_id,
secret=aad_secret,
tenant=aad_tenant
)
resource_management_client = ResourceManagementClient(
credentials=creds,
subscription_id=subscription_id
)
resource = resource_management_client \
.resource_groups.create_or_update(rg, {
'location': location
})
batchai_client = batchai.BatchAIManagementClient(
credentials=creds,
subscription_id=subscription_id
)
return batchai_client
import zipfile
from azure.storage.file import FileService
from azure.storage.blob import BlockBlobService
import azure.mgmt.batchai.models as models
import azure.mgmt.batchai as batchai
from azure.common.credentials import ServicePrincipalCredentials
tenant = "<Insert Correct Value Here>"
subscription = "<Insert Correct Value Here>"
resource_group_name = "batchai"
credentials = ServicePrincipalCredentials(client_id="<Insert Correct Value Here>",
secret="<Insert Correct Value Here>",
token_uri="https://login.microsoftonline.com/{0}/oauth2/token".format(tenant))
client = batchai.BatchAIManagementClient(
credentials=credentials,
subscription_id=subscription,
base_url="")
from azure.mgmt.resource import ResourceManagementClient
resource_management_client = ResourceManagementClient(credentials=credentials, subscription_id=subscription)
group = resource_management_client.resource_groups.create_or_update(
resource_group_name, {'location': 'northeurope'})
from azure.storage.file import FileService
storage_account_name = "<Insert Correct Value Here>"
storage_account_key = "<Insert Correct Value Here>"
fileshare = "data"
