def update_cluster_create_parameters_with_env_variables(params):
"""Replaces placeholders with information from the environment variables.
Currently we support replacing of storage account name and key in mount volumes.
:param models.ClusterCreateParameters params: cluster creation parameters to patch.
"""
storage_account_name = az_config.get(
'batchai',
'storage_account',
fallback=AZURE_BATCHAI_STORAGE_ACCOUNT_PLACEHOLDER)
storage_account_key = az_config.get(
'batchai',
'storage_key',
fallback=AZURE_BATCHAI_STORAGE_KEY_PLACEHOLDER)
# Patch parameters of azure file share.
if params.node_setup and \
params.node_setup.mount_volumes and \
params.node_setup.mount_volumes.azure_file_shares:
for ref in params.node_setup.mount_volumes.azure_file_shares:
if ref.account_name == AZURE_BATCHAI_STORAGE_ACCOUNT_PLACEHOLDER:
ref.account_name = storage_account_name
if ref.azure_file_url and AZURE_BATCHAI_STORAGE_ACCOUNT_PLACEHOLDER in ref.azure_file_url:
ref.azure_file_url = ref.azure_file_url.replace(
AZURE_BATCHAI_STORAGE_ACCOUNT_PLACEHOLDER,
storage_account_name)
if ref.credentials_info and ref.credentials_info.account_key == AZURE_BATCHAI_STORAGE_KEY_PLACEHOLDER:
ref.credentials_info.account_key = storage_account_key
if not ref.credentials_info:
ref.credentials_info = models.AzureStorageCredentialsInfo(
account_key=storage_account_key)
# Verify that all placeholders are replaced with real values.
if (ref.account_name == AZURE_BATCHAI_STORAGE_ACCOUNT_PLACEHOLDER
or AZURE_BATCHAI_STORAGE_ACCOUNT_PLACEHOLDER
in ref.azure_file_url):
raise CLIError(MSG_CONFIGURE_STORAGE_ACCOUNT)
if ref.credentials_info.account_key == AZURE_BATCHAI_STORAGE_KEY_PLACEHOLDER:
raise CLIError(MSG_CONFIGURE_STORAGE_KEY)
# Patch parameters of blob file system.
if params.node_setup and \
params.node_setup.mount_volumes and \
params.node_setup.mount_volumes.azure_blob_file_systems:
for ref in params.node_setup.mount_volumes.azure_blob_file_systems:
if ref.account_name == AZURE_BATCHAI_STORAGE_ACCOUNT_PLACEHOLDER:
ref.account_name = storage_account_name
if ref.credentials_info and ref.credentials_info.account_key == \
AZURE_BATCHAI_STORAGE_KEY_PLACEHOLDER:
ref.credentials_info.account_key = storage_account_key
if not ref.credentials_info:
ref.credentials_info = models.AzureStorageCredentialsInfo(
account_key=storage_account_key)
# Verify that all placeholders are replaced with real values.
if ref.account_name == AZURE_BATCHAI_STORAGE_ACCOUNT_PLACEHOLDER:
raise CLIError(MSG_CONFIGURE_STORAGE_ACCOUNT)
if ref.credentials_info.account_key == AZURE_BATCHAI_STORAGE_KEY_PLACEHOLDER:
raise CLIError(MSG_CONFIGURE_STORAGE_KEY)
def _update_cluster_create_parameters_with_env_variables(cli_ctx, params, account_name=None, account_key=None):
"""Replaces placeholders with information from the environment variables.
Currently we support replacing of storage account name and key in mount volumes.
:param models.ClusterCreateParameters params: cluster creation parameters to patch.
:param str or None account_name: name of the storage account provided as command line argument.
:param str or None account_key: storage account key provided as command line argument.
:return models.ClusterCreateParameters: updated parameters.
"""
result = copy.deepcopy(params)
storage_account_name, storage_account_key = _get_effective_storage_account_name_and_key(
cli_ctx, account_name, account_key)
require_storage_account = False
require_storage_account_key = False
# Patch parameters of azure file share.
if result.node_setup and \
result.node_setup.mount_volumes and \
result.node_setup.mount_volumes.azure_file_shares:
for ref in result.node_setup.mount_volumes.azure_file_shares:
if ref.account_name == AZURE_BATCHAI_STORAGE_ACCOUNT_PLACEHOLDER:
require_storage_account = True
ref.account_name = storage_account_name
if ref.azure_file_url and AZURE_BATCHAI_STORAGE_ACCOUNT_PLACEHOLDER in ref.azure_file_url:
require_storage_account = True
ref.azure_file_url = ref.azure_file_url.replace(
AZURE_BATCHAI_STORAGE_ACCOUNT_PLACEHOLDER, storage_account_name)
if ref.credentials and ref.credentials.account_key == AZURE_BATCHAI_STORAGE_KEY_PLACEHOLDER:
require_storage_account_key = True
ref.credentials.account_key = storage_account_key
if not ref.credentials:
require_storage_account_key = True
ref.credentials = models.AzureStorageCredentialsInfo(account_key=storage_account_key)
# Patch parameters of blob file system.
if result.node_setup and \
result.node_setup.mount_volumes and \
result.node_setup.mount_volumes.azure_blob_file_systems:
for ref in result.node_setup.mount_volumes.azure_blob_file_systems:
if ref.account_name == AZURE_BATCHAI_STORAGE_ACCOUNT_PLACEHOLDER:
require_storage_account = True
ref.account_name = storage_account_name
if ref.credentials and ref.credentials.account_key == AZURE_BATCHAI_STORAGE_KEY_PLACEHOLDER:
require_storage_account_key = True
ref.credentials.account_key = storage_account_key
if not ref.credentials:
require_storage_account_key = True
ref.credentials = models.AzureStorageCredentialsInfo(account_key=storage_account_key)
if require_storage_account and not storage_account_name:
raise CLIError(MSG_CONFIGURE_STORAGE_ACCOUNT)
if require_storage_account_key and not storage_account_key:
raise CLIError(MSG_CONFIGURE_STORAGE_KEY)
return result
def _add_azure_container_to_mount_volumes(cli_ctx, volumes, container_name, mount_path, account_name=None,
account_key=None):
"""Add Azure Storage container to the mount volumes.
:param model.MountVolumes: existing mount volumes.
:param str container_name: container name.
:param str mount_path: relative mount path for the container.
:param str or None account_name: storage account name provided as command line argument.
:param str or None account_key: storage account key provided as command line argument.
:return models.ClusterCreateParameters: updated parameters.
"""
result = copy.deepcopy(volumes) if volumes else models.MountVolumes()
if not mount_path:
raise CLIError('Azure Storage Container relative mount path cannot be empty.')
if result.azure_blob_file_systems is None:
result.azure_blob_file_systems = []
storage_account_name, storage_account_key = _get_effective_storage_account_name_and_key(cli_ctx, account_name,
account_key)
if not storage_account_name:
raise CLIError(MSG_CONFIGURE_STORAGE_ACCOUNT)
if not storage_account_key:
raise CLIError(MSG_CONFIGURE_STORAGE_KEY)
result.azure_blob_file_systems.append(models.AzureBlobFileSystemReference(
relative_mount_path=mount_path,
account_name=storage_account_name,
container_name=container_name,
credentials=models.AzureStorageCredentialsInfo(account_key=storage_account_key)))
return result
def _add_azure_file_share_to_mount_volumes(cli_ctx, volumes, azure_file_share, mount_path, account_name=None,
account_key=None):
"""Add Azure File share to the mount volumes.
:param model.MountVolumes volumes: existing mount volumes.
:param str azure_file_share: name of the azure file share.
:param str mount_path: relative mount path for Azure File share.
:param str or None account_name: storage account name provided as command line argument.
:param str or None account_key: storage account key provided as command line argument.
:return models.ClusterCreateParameters: updated parameters.
"""
result = copy.deepcopy(volumes) if volumes else models.MountVolumes()
if not mount_path:
raise CLIError('Azure File share relative mount path cannot be empty.')
if result.azure_file_shares is None:
result.azure_file_shares = []
effective_account_name, effective_account_key = _get_effective_storage_account_name_and_key(cli_ctx, account_name,
account_key)
if not effective_account_name:
raise CLIError(MSG_CONFIGURE_STORAGE_ACCOUNT)
if not effective_account_key:
raise CLIError(MSG_CONFIGURE_STORAGE_KEY)
result.azure_file_shares.append(models.AzureFileShareReference(
relative_mount_path=mount_path,
account_name=effective_account_name,
azure_file_url=_get_azure_file_url(cli_ctx, effective_account_name, azure_file_share),
credentials=models.AzureStorageCredentialsInfo(account_key=effective_account_key)))
return result
def _add_azure_file_share_to_cluster_create_parameters(cli_ctx, params, azure_file_share, mount_path, account_name=None,
account_key=None):
"""Add Azure File share to the cluster create parameters.
:param model.ClusterCreateParameters params: cluster create parameters.
:param str azure_file_share: name of the azure file share.
:param str mount_path: relative mount path for Azure File share.
:param str or None account_name: storage account name provided as command line argument.
:param str or None account_key: storage account key provided as command line argument.
:return models.ClusterCreateParameters: updated parameters.
"""
result = copy.deepcopy(params)
if not mount_path:
raise CLIError('Azure File share relative mount path cannot be empty.')
if result.node_setup is None:
result.node_setup = models.NodeSetup()
if result.node_setup.mount_volumes is None:
result.node_setup.mount_volumes = models.MountVolumes()
if result.node_setup.mount_volumes.azure_file_shares is None:
result.node_setup.mount_volumes.azure_file_shares = []
storage_account_name, storage_account_key = _get_effective_storage_account_name_and_key(cli_ctx, account_name,
account_key)
if not storage_account_name:
raise CLIError(MSG_CONFIGURE_STORAGE_ACCOUNT)
if not storage_account_key:
raise CLIError(MSG_CONFIGURE_STORAGE_KEY)
result.node_setup.mount_volumes.azure_file_shares.append(models.AzureFileShareReference(
relative_mount_path=mount_path,
account_name=storage_account_name,
azure_file_url='https://{0}.file.core.windows.net/{1}'.format(storage_account_name, azure_file_share),
credentials=models.AzureStorageCredentialsInfo(storage_account_key)))
return result
def add_azure_container_to_cluster_create_parameters(params, container_name,
mount_path):
"""Add Azure Storage container to the cluster create parameters.
:param model.ClusterCreateParameters params: cluster create parameters.
:param str container_name: container name.
:param str mount_path: relative mount path for the container.
"""
if not mount_path:
raise CLIError(
'Azure Storage container relative mount path cannot be empty.')
if params.node_setup is None:
params.node_setup = models.NodeSetup()
if params.node_setup.mount_volumes is None:
params.node_setup.mount_volumes = models.MountVolumes()
if params.node_setup.mount_volumes.azure_blob_file_systems is None:
params.node_setup.mount_volumes.azure_blob_file_systems = []
storage_account_name = az_config.get('batchai',
'storage_account',
fallback=None)
if not storage_account_name:
raise CLIError(MSG_CONFIGURE_STORAGE_ACCOUNT)
storage_account_key = az_config.get('batchai',
'storage_key',
fallback=None)
if not storage_account_key:
raise CLIError(MSG_CONFIGURE_STORAGE_KEY)
params.node_setup.mount_volumes.azure_blob_file_systems.append(
models.AzureBlobFileSystemReference(
relative_mount_path=mount_path,
account_name=storage_account_name,
container_name=container_name,
credentials=models.AzureStorageCredentialsInfo(
account_key=storage_account_key)))
def add_azure_file_share_to_cluster_create_parameters(params, azure_file_share,
mount_path):
"""Add Azure File share to the cluster create parameters.
:param model.ClusterCreateParameters params: cluster create parameters.
:param str azure_file_share: name of the azure file share.
:param str mount_path: relative mount path for Azure File share.
"""
if not mount_path:
raise CLIError('Azure File share relative mount path cannot be empty.')
if params.node_setup is None:
params.node_setup = models.NodeSetup()
if params.node_setup.mount_volumes is None:
params.node_setup.mount_volumes = models.MountVolumes()
if params.node_setup.mount_volumes.azure_file_shares is None:
params.node_setup.mount_volumes.azure_file_shares = []
storage_account_name = az_config.get('batchai',
'storage_account',
fallback=None)
if not storage_account_name:
raise CLIError(MSG_CONFIGURE_STORAGE_ACCOUNT)
storage_account_key = az_config.get('batchai',
'storage_key',
fallback=None)
if not storage_account_key:
raise CLIError(MSG_CONFIGURE_STORAGE_KEY)
params.node_setup.mount_volumes.azure_file_shares.append(
models.AzureFileShareReference(
relative_mount_path=mount_path,
account_name=storage_account_name,
azure_file_url='https://{0}.file.core.windows.net/{1}'.format(
storage_account_name, azure_file_share),
credentials=models.AzureStorageCredentialsInfo(
storage_account_key)))
def prepare_batch_ai_workspace(client, service, config):
# Create Batch AI workspace
client.workspaces.create(config.workspace_resource_group,
config.workspace,
config.location)
# Create GPU cluster
parameters = models.ClusterCreateParameters(
# VM size. Use N-series for GPU
vm_size=config.workspace_vm_size,
# Configure the ssh users
user_account_settings=models.UserAccountSettings(
admin_user_name=config.admin,
admin_user_password=config.admin_password),
# Number of VMs in the cluster
scale_settings=models.ScaleSettings(
manual=models.ManualScaleSettings(target_node_count=config.workspace_node_count)
),
# Configure each node in the cluster
node_setup=models.NodeSetup(
# Mount shared volumes to the host
mount_volumes=models.MountVolumes(
azure_file_shares=[
models.AzureFileShareReference(
account_name=config.storage_account_name,
credentials=models.AzureStorageCredentialsInfo(
account_key=config.storage_account_key),
azure_file_url='https://{0}/{1}'.format(
service.primary_endpoint, config.workspace_file_share),
relative_mount_path=config.workspace_relative_mount_path)],
),
),
)
client.clusters.create(config.workspace_resource_group, config.workspace, config.workspace_cluster, parameters).result()
def _get_effective_credentials(cli_ctx, existing_credentials, account_name):
"""Returns AzureStorageCredentialInfo for the account
:param models.AzureStorageCredentialsInfo existing_credentials: known credentials
:param str account_name: storage account name
:return models.AzureStorageCredentialsInfo: credentials to be used
"""
if existing_credentials and (existing_credentials.account_key or existing_credentials.account_key_secret_reference):
return existing_credentials
return models.AzureStorageCredentialsInfo(
account_key=_get_storage_account_key(cli_ctx, account_name, account_key=None))
def create_volume(storage_name, storage_key, azure_file_share_name,
azure_file_share):
return models.MountVolumes(azure_file_shares=[
models.AzureFileShareReference(
account_name=storage_name,
credentials=models.AzureStorageCredentialsInfo(
account_key=storage_key),
azure_file_url='https://{0}.file.core.windows.net/{1}'.format(
storage_name, azure_file_share_name),
relative_mount_path=azure_file_share)
])
def _patch_mount_volumes(cli_ctx, volumes, account_name=None, account_key=None):
"""Patches mount volumes by replacing placeholders and adding credentials information.
:param models.MountVolumes or None volumes: mount volumes.
:param str or None account_name: name of the storage account provided as command line argument.
:param str or None account_key: storage account key provided as command line argument.
:return models.ClusterCreateParameters: updated parameters.
"""
if volumes is None:
return None
result = copy.deepcopy(volumes) # type: models.MountVolumes
storage_account_name, storage_account_key = _get_effective_storage_account_name_and_key(
cli_ctx, account_name, account_key)
require_storage_account = False
require_storage_account_key = False
# Patch parameters of azure file share.
if result.azure_file_shares:
for ref in result.azure_file_shares:
# Populate account name if it was not provided
if not ref.account_name:
ref.account_name = _get_account_name_from_azure_file_url(ref.azure_file_url)
# Replace placeholders
if ref.account_name == AZURE_BATCHAI_STORAGE_ACCOUNT_PLACEHOLDER:
require_storage_account = True
ref.account_name = storage_account_name
if ref.azure_file_url and AZURE_BATCHAI_STORAGE_ACCOUNT_PLACEHOLDER in ref.azure_file_url:
require_storage_account = True
ref.azure_file_url = ref.azure_file_url.replace(
AZURE_BATCHAI_STORAGE_ACCOUNT_PLACEHOLDER, storage_account_name)
if ref.credentials and ref.credentials.account_key == AZURE_BATCHAI_STORAGE_KEY_PLACEHOLDER:
require_storage_account_key = True
ref.credentials.account_key = storage_account_key
if not ref.credentials and ref.account_name == storage_account_name:
require_storage_account_key = True
ref.credentials = models.AzureStorageCredentialsInfo(account_key=storage_account_key)
ref.credentials = _get_effective_credentials(cli_ctx, ref.credentials, ref.account_name)
# Patch parameters of blob file systems.
if result.azure_blob_file_systems:
for ref in result.azure_blob_file_systems:
# Replace placeholders
if ref.account_name == AZURE_BATCHAI_STORAGE_ACCOUNT_PLACEHOLDER:
require_storage_account = True
ref.account_name = storage_account_name
if ref.credentials and ref.credentials.account_key == AZURE_BATCHAI_STORAGE_KEY_PLACEHOLDER:
require_storage_account_key = True
ref.credentials.account_key = storage_account_key
if not ref.credentials and ref.account_name == storage_account_name:
require_storage_account_key = True
ref.credentials = models.AzureStorageCredentialsInfo(account_key=storage_account_key)
# Populate the rest of credentials based on the account name
if not ref.account_name:
raise CLIError('Ill-formed Azure Blob File System reference in the configuration file - no account '
'name provided.')
ref.credentials = _get_effective_credentials(cli_ctx, ref.credentials, ref.account_name)
if require_storage_account and not storage_account_name:
raise CLIError(MSG_CONFIGURE_STORAGE_ACCOUNT)
if require_storage_account_key and not storage_account_key:
raise CLIError(MSG_CONFIGURE_STORAGE_KEY)
return result
def create_cluster(client,
location,
resource_group,
cluster_name,
vm_size,
target_nodes,
storage_account,
storage_account_key,
file_servers=None,
file_systems=None,
subnet_id=None,
setup_task_cmd=None,
setup_task_env=None,
setup_task_secrets=None):
"""Creates a cluster with given parameters and mounted Azure Files
:param BatchAIManagementClient client: client instance.
:param str location: location.
:param str resource_group: resource group name.
:param str cluster_name: name of the cluster.
:param str vm_size: vm size.
:param int target_nodes: number of nodes.
:param str storage_account: name of the storage account.
:param str storage_account_key: storage account key.
:param list(models.FileServerReference) file_servers: file servers.
:param list(models.UnmanagedFileServerReference) file_systems: file systems.
:param str setup_task_cmd: start task cmd line.
:param dict[str, str] setup_task_env: environment variables for start task.
:param dict[str, str] setup_task_secrets: environment variables with secret values for start task, server doesn't
return values for these environment variables in get cluster responses.
:param str subnet_id: virtual network subnet id.
:return models.Cluster: the created cluster
"""
Helpers._create_file_share(storage_account, storage_account_key)
setup_task = None
if setup_task_cmd:
setup_task = models.SetupTask(
command_line=setup_task_cmd,
environment_variables=[
models.EnvironmentVariable(name=k, value=v)
for k, v in setup_task_env.items()
],
secrets=[
models.EnvironmentVariableWithSecretValue(name=k, value=v)
for k, v in setup_task_secrets.items()
],
std_out_err_path_prefix='$AZ_BATCHAI_MOUNT_ROOT/{0}'.format(
Helpers.AZURE_FILES_MOUNTING_PATH))
client.workspaces.create(resource_group,
Helpers.DEFAULT_WORKSPACE_NAME,
location).result()
return client.clusters.create(
resource_group,
Helpers.DEFAULT_WORKSPACE_NAME,
cluster_name,
parameters=models.ClusterCreateParameters(
vm_size=vm_size,
scale_settings=models.ScaleSettings(
manual=models.ManualScaleSettings(
target_node_count=target_nodes)),
node_setup=models.NodeSetup(
mount_volumes=models.MountVolumes(
azure_file_shares=[
models.AzureFileShareReference(
azure_file_url=
'https://{0}.file.core.windows.net/{1}'.format(
storage_account, Helpers.AZURE_FILES_NAME),
relative_mount_path=Helpers.
AZURE_FILES_MOUNTING_PATH,
account_name=storage_account,
credentials=models.AzureStorageCredentialsInfo(
account_key=storage_account_key),
)
],
file_servers=file_servers,
unmanaged_file_systems=file_systems),
setup_task=setup_task),
subnet=subnet_id,
user_account_settings=models.UserAccountSettings(
admin_user_name=Helpers.ADMIN_USER_NAME,
admin_user_password=Helpers.ADMIN_USER_PASSWORD),
vm_priority='lowpriority')).result()
command_line=
'python $AZ_BATCHAI_JOB_MOUNT_ROOT/resources/scripts/FF_multi_step_multivariate.py \
--scriptdir $AZ_BATCHAI_JOB_MOUNT_ROOT/resources/scripts \
--datadir $AZ_BATCHAI_JOB_MOUNT_ROOT/resources/data \
--outdir $AZ_BATCHAI_OUTPUT_ALL \
-l {0} -n {1} -b {2} -T {3} -r {4} -a {5}'.format(
parameters['LATENT_DIM'], parameters['HIDDEN_LAYERS'],
parameters['BATCH_SIZE'], parameters['T'],
parameters['LEARNING_RATE'], parameters['ALPHA'])),
container_settings=models.ContainerSettings(
image_source_registry=models.ImageSourceRegistry(
image=cfg['docker_image'])),
mount_volumes=models.MountVolumes(azure_file_shares=[
models.AzureFileShareReference(
account_name=cfg['storage_account']['name'],
credentials=models.AzureStorageCredentialsInfo(
account_key=cfg['storage_account']['key']),
azure_file_url='https://' + cfg['storage_account']['name'] +
'.file.core.windows.net/logs',
relative_mount_path='logs'),
models.AzureFileShareReference(
account_name=cfg['storage_account']['name'],
credentials=models.AzureStorageCredentialsInfo(
account_key=cfg['storage_account']['key']),
azure_file_url='https://' + cfg['storage_account']['name'] +
'.file.core.windows.net/resources',
relative_mount_path='resources'),
models.AzureFileShareReference(
account_name=cfg['storage_account']['name'],
credentials=models.AzureStorageCredentialsInfo(
account_key=cfg['storage_account']['key']),
azure_file_url='https://' + cfg['storage_account']['name'] +
parameters = models.ClusterCreateParameters(
location='northeurope',
vm_size='STANDARD_NC6',
user_account_settings=models.UserAccountSettings(
admin_user_name="shwars",
admin_user_password="******"),
scale_settings=models.ScaleSettings(
manual=models.ManualScaleSettings(target_node_count=1)
),
node_setup=models.NodeSetup(
# Mount shared volumes to the host
mount_volumes=models.MountVolumes(
azure_file_shares=[
models.AzureFileShareReference(
account_name=storage_account_name,
credentials=models.AzureStorageCredentialsInfo(
account_key=storage_account_key),
azure_file_url='https://{0}.file.core.windows.net/{1}'.format(
storage_account_name, fileshare),
relative_mount_path = relative_mount_point)],
),
),
)
client.clusters.create(resource_group_name, cluster_name, parameters).result()
cluster = client.clusters.get(resource_group_name, cluster_name)
print('Cluster state: {0} Target: {1}; Allocated: {2}; Idle: {3}; '
'Unusable: {4}; Running: {5}; Preparing: {6}; leaving: {7}'.format(
cluster.allocation_state,
cluster.scale_settings.manual.target_node_count,
cluster.current_node_count,
def test_job_level_mounting(self, resource_group, location, cluster,
storage_account, storage_account_key):
"""Tests if it's possible to mount external file systems for a job."""
job_name = 'job'
# Create file share and container to mount on the job level
if storage_account.name != FAKE_STORAGE.name:
files = FileService(storage_account.name, storage_account_key)
files.create_share('jobshare', fail_on_exist=False)
blobs = BlockBlobService(storage_account.name, storage_account_key)
blobs.create_container('jobcontainer', fail_on_exist=False)
job = self.client.jobs.create(
resource_group.name,
job_name,
parameters=models.JobCreateParameters(
location=location,
cluster=models.ResourceId(id=cluster.id),
node_count=1,
mount_volumes=models.
MountVolumes(azure_file_shares=[
models.AzureFileShareReference(
account_name=storage_account.name,
azure_file_url='https://{0}.file.core.windows.net/{1}'.
format(storage_account.name, 'jobshare'),
relative_mount_path='job_afs',
credentials=models.AzureStorageCredentialsInfo(
account_key=storage_account_key),
)
],
azure_blob_file_systems=[
models.AzureBlobFileSystemReference(
account_name=storage_account.name,
container_name='jobcontainer',
relative_mount_path='job_bfs',
credentials=models.
AzureStorageCredentialsInfo(
account_key=storage_account_key),
)
]),
# Put standard output on cluster level AFS to check that the job has access to it.
std_out_err_path_prefix='$AZ_BATCHAI_MOUNT_ROOT/{0}'.format(
AZURE_FILES_MOUNTING_PATH),
# Create two output directories on job level AFS and blobfuse.
output_directories=[
models.OutputDirectory(
id='OUTPUT1',
path_prefix='$AZ_BATCHAI_JOB_MOUNT_ROOT/job_afs'),
models.OutputDirectory(
id='OUTPUT2',
path_prefix='$AZ_BATCHAI_JOB_MOUNT_ROOT/job_bfs')
],
# Check that the job preparation has access to job level file systems.
job_preparation=models.JobPreparation(
command_line=
'echo afs > $AZ_BATCHAI_OUTPUT_OUTPUT1/prep_afs.txt; '
'echo bfs > $AZ_BATCHAI_OUTPUT_OUTPUT2/prep_bfs.txt; '
'echo done'),
# Check that the job has access to job
custom_toolkit_settings=models.CustomToolkitSettings(
command_line=
'echo afs > $AZ_BATCHAI_OUTPUT_OUTPUT1/job_afs.txt; '
'echo bfs > $AZ_BATCHAI_OUTPUT_OUTPUT2/job_bfs.txt; '
'mkdir $AZ_BATCHAI_OUTPUT_OUTPUT1/afs; '
'echo afs > $AZ_BATCHAI_OUTPUT_OUTPUT1/afs/job_afs.txt; '
'mkdir $AZ_BATCHAI_OUTPUT_OUTPUT2/bfs; '
'echo bfs > $AZ_BATCHAI_OUTPUT_OUTPUT2/bfs/job_bfs.txt; '
'echo done'))).result()
self.assertEqual(
wait_for_job_completion(self.is_live, self.client,
resource_group.name, job.name, MINUTE),
models.ExecutionState.succeeded)
job = self.client.jobs.get(resource_group.name, job.name)
# Assert job and job prep standard output is populated on cluster level filesystem
assert_job_files_are(
self, self.client, resource_group.name, job.name,
STANDARD_OUTPUT_DIRECTORY_ID, {
u'stdout.txt': u'done\n',
u'stderr.txt': u'',
u'stdout-job_prep.txt': u'done\n',
u'stderr-job_prep.txt': u''
})
# Assert files are generated on job level AFS
assert_job_files_are(self, self.client, resource_group.name, job.name,
'OUTPUT1', {
u'job_afs.txt': u'afs\n',
u'prep_afs.txt': u'afs\n',
u'afs': None
})
# Assert files are generated on job level blobfuse
assert_job_files_are(self, self.client, resource_group.name, job.name,
'OUTPUT2', {
u'job_bfs.txt': u'bfs\n',
u'prep_bfs.txt': u'bfs\n',
u'bfs': None
})
# Assert subfolders are available via API
assert_job_files_in_path_are(self, self.client, resource_group.name,
job.name, 'OUTPUT1', 'afs',
{u'job_afs.txt': u'afs\n'})
assert_job_files_in_path_are(self, self.client, resource_group.name,
job.name, 'OUTPUT2', 'bfs',
{u'job_bfs.txt': u'bfs\n'})
# Assert that we can access the output files created on job level mount volumes directly in storage using path
# segment returned by the server.
if storage_account.name != FAKE_STORAGE.name:
files = FileService(storage_account.name, storage_account_key)
self.assertTrue(
files.exists(
'jobshare', job.job_output_directory_path_segment + '/' +
OUTPUT_DIRECTORIES_FOLDER_NAME, 'job_afs.txt'))
blobs = BlockBlobService(storage_account.name, storage_account_key)
self.assertTrue(
blobs.exists(
'jobcontainer', job.job_output_directory_path_segment +
'/' + OUTPUT_DIRECTORIES_FOLDER_NAME + '/job_bfs.txt'))
# After the job is done the filesystems should be unmounted automatically, check this by submitting a new job.
checker = self.client.jobs.create(
resource_group.name,
'checker',
parameters=models.JobCreateParameters(
location=location,
cluster=models.ResourceId(id=cluster.id),
node_count=1,
std_out_err_path_prefix='$AZ_BATCHAI_MOUNT_ROOT/{0}'.format(
AZURE_FILES_MOUNTING_PATH),
custom_toolkit_settings=models.CustomToolkitSettings(
command_line='echo job; df | grep -E "job_bfs|job_afs"'))
).result()
# Check the job failed because there are not job level mount volumes anymore
self.assertEqual(
wait_for_job_completion(self.is_live, self.client,
resource_group.name, checker.name, MINUTE),
models.ExecutionState.failed)
# Check that the cluster level AFS was still mounted
assert_job_files_are(self, self.client, resource_group.name,
checker.name, STANDARD_OUTPUT_DIRECTORY_ID, {
u'stdout.txt': u'job\n',
u'stderr.txt': u''
})
