def test_job_environment_variables_and_secrets(self, resource_group,
location, cluster):
"""Tests if it's possible to mount external file systems for a job."""
job_name = 'job'
job = self.client.jobs.create(
resource_group.name,
helpers.DEFAULT_WORKSPACE_NAME,
helpers.DEFAULT_EXPERIMENT_NAME,
job_name,
parameters=models.JobCreateParameters(
cluster=models.ResourceId(id=cluster.id),
node_count=1,
std_out_err_path_prefix='$AZ_BATCHAI_MOUNT_ROOT/{0}'.format(
helpers.AZURE_FILES_MOUNTING_PATH),
environment_variables=[
models.EnvironmentVariable(name='VARIABLE', value='VALUE')
],
secrets=[
models.EnvironmentVariableWithSecretValue(
name='SECRET_VARIABLE', value='SECRET')
],
# Check that the job preparation has access to env variables and secrets.
job_preparation=models.JobPreparation(
command_line='echo $VARIABLE $SECRET_VARIABLE'),
# Check that the job has access to env variables and secrets.
custom_toolkit_settings=models.CustomToolkitSettings(
command_line='echo $VARIABLE $SECRET_VARIABLE'))).result(
) # type: models.Job
self.assertEqual(
helpers.wait_for_job_completion(self.is_live, self.client,
resource_group.name, job.name,
helpers.MINUTE),
models.ExecutionState.succeeded)
# Check that environment variables are reported by the server.
self.assertEqual(len(job.environment_variables), 1)
self.assertEqual(job.environment_variables[0].name, 'VARIABLE')
self.assertEqual(job.environment_variables[0].value, 'VALUE')
# Check that secrets are reported back by server, but value is not reported.
self.assertEqual(len(job.secrets), 1)
self.assertEqual(job.secrets[0].name, 'SECRET_VARIABLE')
self.assertIsNone(job.secrets[0].value)
# Check that job and job prep had access to the env variables and secrets.
helpers.assert_job_files_are(
self, self.client, resource_group.name, job.name,
helpers.STANDARD_OUTPUT_DIRECTORY_ID, {
u'stdout.txt': u'VALUE SECRET\n',
u'stderr.txt': u'',
u'stdout-job_prep.txt': u'VALUE SECRET\n',
u'stderr-job_prep.txt': u''
})
def _generate_jobs(self, job_create_parameters, num_jobs=None):
"""
Creates copies of job_create_parameters with the template strings
and Substitution objects substituted with combinations of parameters
specified in param_specs.
:param job_create_parameters: an instance of JobCreateParameters
:param num_jobs: the number of jobs to generate with random search. If
None, grid search will be performed.
"""
jcps = []
for param_dict in self._generate_param_dicts(num_jobs):
jcp_substituted = self._substitute_params(job_create_parameters,
param_dict)
environment_variables = [
models.EnvironmentVariable(name=parameter_name,
value=str(value))
for parameter_name, value in param_dict.items()
]
jcp_substituted.environment_variables = environment_variables
jcps.append(jcp_substituted)
return jcps
def create_cluster(client,
location,
resource_group,
cluster_name,
vm_size,
target_nodes,
storage_account,
storage_account_key,
file_servers=None,
file_systems=None,
subnet_id=None,
setup_task_cmd=None,
setup_task_env=None,
setup_task_secrets=None):
"""Creates a cluster with given parameters and mounted Azure Files
:param BatchAIManagementClient client: client instance.
:param str location: location.
:param str resource_group: resource group name.
:param str cluster_name: name of the cluster.
:param str vm_size: vm size.
:param int target_nodes: number of nodes.
:param str storage_account: name of the storage account.
:param str storage_account_key: storage account key.
:param list(models.FileServerReference) file_servers: file servers.
:param list(models.UnmanagedFileServerReference) file_systems: file systems.
:param str setup_task_cmd: start task cmd line.
:param dict[str, str] setup_task_env: environment variables for start task.
:param dict[str, str] setup_task_secrets: environment variables with secret values for start task, server doesn't
return values for these environment variables in get cluster responses.
:param str subnet_id: virtual network subnet id.
:return models.Cluster: the created cluster
"""
Helpers._create_file_share(storage_account, storage_account_key)
setup_task = None
if setup_task_cmd:
setup_task = models.SetupTask(
command_line=setup_task_cmd,
environment_variables=[
models.EnvironmentVariable(name=k, value=v)
for k, v in setup_task_env.items()
],
secrets=[
models.EnvironmentVariableWithSecretValue(name=k, value=v)
for k, v in setup_task_secrets.items()
],
std_out_err_path_prefix='$AZ_BATCHAI_MOUNT_ROOT/{0}'.format(
Helpers.AZURE_FILES_MOUNTING_PATH))
client.workspaces.create(resource_group,
Helpers.DEFAULT_WORKSPACE_NAME,
location).result()
return client.clusters.create(
resource_group,
Helpers.DEFAULT_WORKSPACE_NAME,
cluster_name,
parameters=models.ClusterCreateParameters(
vm_size=vm_size,
scale_settings=models.ScaleSettings(
manual=models.ManualScaleSettings(
target_node_count=target_nodes)),
node_setup=models.NodeSetup(
mount_volumes=models.MountVolumes(
azure_file_shares=[
models.AzureFileShareReference(
azure_file_url=
'https://{0}.file.core.windows.net/{1}'.format(
storage_account, Helpers.AZURE_FILES_NAME),
relative_mount_path=Helpers.
AZURE_FILES_MOUNTING_PATH,
account_name=storage_account,
credentials=models.AzureStorageCredentialsInfo(
account_key=storage_account_key),
)
],
file_servers=file_servers,
unmanaged_file_systems=file_systems),
setup_task=setup_task),
subnet=subnet_id,
user_account_settings=models.UserAccountSettings(
admin_user_name=Helpers.ADMIN_USER_NAME,
admin_user_password=Helpers.ADMIN_USER_PASSWORD),
vm_priority='lowpriority')).result()