def __create_pool_and_job(self, cluster_conf: models.ClusterConfiguration, software_metadata_key: str, start_task, VmImageModel):
"""
Create a pool and job
:param cluster_conf: the configuration object used to create the cluster
:type cluster_conf: aztk.models.ClusterConfiguration
:parm software_metadata_key: the id of the software being used on the cluster
:param start_task: the start task for the cluster
:param VmImageModel: the type of image to provision for the cluster
:param wait: wait until the cluster is ready
"""
self._get_cluster_data(cluster_conf.cluster_id).save_cluster_config(cluster_conf)
# reuse pool_id as job_id
pool_id = cluster_conf.cluster_id
job_id = cluster_conf.cluster_id
# Get a verified node agent sku
sku_to_use, image_ref_to_use = \
helpers.select_latest_verified_vm_image_with_node_agent_sku(
VmImageModel.publisher, VmImageModel.offer, VmImageModel.sku, self.batch_client)
network_conf = None
if cluster_conf.subnet_id is not None:
network_conf = batch_models.NetworkConfiguration(
subnet_id=cluster_conf.subnet_id)
auto_scale_formula = "$TargetDedicatedNodes={0}; $TargetLowPriorityNodes={1}".format(
cluster_conf.vm_count, cluster_conf.vm_low_pri_count)
# Confiure the pool
pool = batch_models.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batch_models.VirtualMachineConfiguration(
image_reference=image_ref_to_use,
node_agent_sku_id=sku_to_use),
vm_size=cluster_conf.vm_size,
enable_auto_scale=True,
auto_scale_formula=auto_scale_formula,
auto_scale_evaluation_interval=timedelta(minutes=5),
start_task=start_task,
enable_inter_node_communication=True if not cluster_conf.subnet_id else False,
max_tasks_per_node=4,
network_configuration=network_conf,
metadata=[
batch_models.MetadataItem(
name=constants.AZTK_SOFTWARE_METADATA_KEY, value=software_metadata_key),
batch_models.MetadataItem(
name=constants.AZTK_MODE_METADATA_KEY, value=constants.AZTK_CLUSTER_MODE_METADATA)
])
# Create the pool + create user for the pool
helpers.create_pool_if_not_exist(pool, self.batch_client)
# Create job
job = batch_models.JobAddParameter(
id=job_id,
pool_info=batch_models.PoolInformation(pool_id=pool_id))
# Add job to batch
self.batch_client.job.add(job)
return helpers.get_cluster(cluster_conf.cluster_id, self.batch_client)
def create_job(batch_service_client, job_id, pool_id):
"""
Creates a job with the specified ID, associated with the specified pool.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str job_id: The ID for the job.
:param str pool_id: The ID for the pool.
"""
print('Creating job [{}]...'.format(job_id))
job = batch.models.JobAddParameter(
job_id,
batch.models.PoolInformation(pool_id=pool_id),
metadata=[
batchmodels.MetadataItem("AZURE_TENANT_ID", _AZURE_TENANT_ID),
batchmodels.MetadataItem("AZURE_SUBSCRIPTION_ID",
_AZURE_SUBSCRIPTION_ID),
batchmodels.MetadataItem("AZURE_CLIENT_ID", _AZURE_CLIENT_ID),
batchmodels.MetadataItem("AZURE_CLIENT_OID", _AZURE_CLIENT_OID),
batchmodels.MetadataItem("AZURE_CLIENT_SECRET",
_AZURE_CLIENT_SECRET)
])
batch_service_client.job.add(job)
def __create_pool_and_job(self, cluster_conf, software_metadata_key: str,
start_task, VmImageModel):
"""
Create a pool and job
:param cluster_conf: the configuration object used to create the cluster
:type cluster_conf: aztk.models.ClusterConfiguration
:parm software_metadata_key: the id of the software being used on the cluster
:param start_task: the start task for the cluster
:param VmImageModel: the type of image to provision for the cluster
:param wait: wait until the cluster is ready
"""
# reuse pool_id as job_id
pool_id = cluster_conf.cluster_id
job_id = cluster_conf.cluster_id
# Get a verified node agent sku
sku_to_use, image_ref_to_use = \
helpers.select_latest_verified_vm_image_with_node_agent_sku(
VmImageModel.publisher, VmImageModel.offer, VmImageModel.sku, self.batch_client)
# Confiure the pool
pool = batch_models.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batch_models.
VirtualMachineConfiguration(image_reference=image_ref_to_use,
node_agent_sku_id=sku_to_use),
vm_size=cluster_conf.vm_size,
target_dedicated_nodes=cluster_conf.vm_count,
target_low_priority_nodes=cluster_conf.vm_low_pri_count,
start_task=start_task,
enable_inter_node_communication=True,
max_tasks_per_node=1,
metadata=[
batch_models.MetadataItem(
name=constants.AZTK_SOFTWARE_METADATA_KEY,
value=software_metadata_key),
])
# Create the pool + create user for the pool
helpers.create_pool_if_not_exist(pool, self.batch_client)
# Create job
job = batch_models.JobAddParameter(
id=job_id, pool_info=batch_models.PoolInformation(pool_id=pool_id))
# Add job to batch
self.batch_client.job.add(job)
return helpers.get_cluster(cluster_conf.cluster_id, self.batch_client)
def test_batch_update_pools(self, **kwargs):
client = self.create_sharedkey_client(**kwargs)
# Test Create Paas Pool
test_paas_pool = models.PoolAddParameter(
id=self.get_resource_name('batch_paas_'),
vm_size='small',
cloud_service_configuration=models.CloudServiceConfiguration(
os_family='5'
),
start_task=models.StartTask(
command_line="cmd.exe /c \"echo hello world\"",
resource_files=[models.ResourceFile('https://blobsource.com', 'filename.txt')],
environment_settings=[models.EnvironmentSetting('ENV_VAR', 'env_value')],
user_identity=models.UserIdentity(
auto_user=models.AutoUserSpecification(
elevation_level=models.ElevationLevel.admin
)
)
)
)
response = client.pool.add(test_paas_pool)
self.assertIsNone(response)
# Test Upgrade Pool OS
self.assertBatchError(
"PoolVersionEqualsUpgradeVersion",
client.pool.upgrade_os,
test_paas_pool.id,
"*"
)
# Test Update Pool Parameters
params = models.PoolUpdatePropertiesParameter([], [], [models.MetadataItem('foo', 'bar')])
response = client.pool.update_properties(test_paas_pool.id, params)
self.assertIsNone(response)
# Test Patch Pool Parameters
params = models.PoolPatchParameter(metadata=[models.MetadataItem('foo2', 'bar2')])
response = client.pool.patch(test_paas_pool.id, params)
self.assertIsNone(response)
# Test Pool Exists
response = client.pool.exists(test_paas_pool.id)
self.assertTrue(response)
# Test Get Pool
pool = client.pool.get(test_paas_pool.id)
self.assertIsInstance(pool, models.CloudPool)
self.assertEqual(pool.id, test_paas_pool.id)
self.assertEqual(pool.state, models.PoolState.active)
self.assertEqual(pool.allocation_state, models.AllocationState.steady)
self.assertEqual(pool.cloud_service_configuration.os_family, '5')
self.assertEqual(pool.vm_size, 'small')
self.assertIsNone(pool.start_task)
self.assertEqual(pool.metadata[0].name, 'foo2')
self.assertEqual(pool.metadata[0].value, 'bar2')
# Test Get Pool with OData Clauses
options = models.PoolGetOptions(select='id,state', expand='stats')
pool = client.pool.get(test_paas_pool.id, options)
self.assertIsInstance(pool, models.CloudPool)
self.assertEqual(pool.id, test_paas_pool.id)
self.assertEqual(pool.state, models.PoolState.active)
self.assertIsNone(pool.allocation_state)
self.assertIsNone(pool.vm_size)
# Test Delete Pool
response = client.pool.delete(test_paas_pool.id)
self.assertIsNone(response)
def __submit_job(self,
job_configuration,
start_task,
job_manager_task,
autoscale_formula,
software_metadata_key: str,
vm_image_model,
application_metadata):
"""
Job Submission
:param job_configuration -> aztk_sdk.spark.models.JobConfiguration
:param start_task -> batch_models.StartTask
:param job_manager_task -> batch_models.TaskAddParameter
:param autoscale forumula -> str
:param software_metadata_key -> str
:param vm_image_model -> aztk_sdk.models.VmImage
:returns None
"""
self._get_cluster_data(job_configuration.id).save_cluster_config(job_configuration.to_cluster_config())
# get a verified node agent sku
sku_to_use, image_ref_to_use = \
helpers.select_latest_verified_vm_image_with_node_agent_sku(
vm_image_model.publisher, vm_image_model.offer, vm_image_model.sku, self.batch_client)
# set up subnet if necessary
network_conf = None
if job_configuration.subnet_id:
network_conf = batch_models.NetworkConfiguration(
subnet_id=job_configuration.subnet_id)
# set up a schedule for a recurring job
auto_pool_specification = batch_models.AutoPoolSpecification(
pool_lifetime_option=batch_models.PoolLifetimeOption.job_schedule,
auto_pool_id_prefix=job_configuration.id,
keep_alive=False,
pool=batch_models.PoolSpecification(
display_name=job_configuration.id,
virtual_machine_configuration=batch_models.VirtualMachineConfiguration(
image_reference=image_ref_to_use,
node_agent_sku_id=sku_to_use),
vm_size=job_configuration.vm_size,
enable_auto_scale=True,
auto_scale_formula=autoscale_formula,
auto_scale_evaluation_interval=timedelta(minutes=5),
start_task=start_task,
enable_inter_node_communication=not job_configuration.mixed_mode(),
network_configuration=network_conf,
max_tasks_per_node=4,
metadata=[
batch_models.MetadataItem(
name=constants.AZTK_SOFTWARE_METADATA_KEY, value=software_metadata_key),
batch_models.MetadataItem(
name=constants.AZTK_MODE_METADATA_KEY, value=constants.AZTK_JOB_MODE_METADATA)
]
)
)
# define job specification
job_spec = batch_models.JobSpecification(
pool_info=batch_models.PoolInformation(auto_pool_specification=auto_pool_specification),
display_name=job_configuration.id,
on_all_tasks_complete=batch_models.OnAllTasksComplete.terminate_job,
job_manager_task=job_manager_task,
metadata=[
batch_models.MetadataItem(
name='applications', value=application_metadata)
]
)
# define schedule
schedule = batch_models.Schedule(
do_not_run_until=None,
do_not_run_after=None,
start_window=None,
recurrence_interval=None
)
# create job schedule and add task
setup = batch_models.JobScheduleAddParameter(
id=job_configuration.id,
schedule=schedule,
job_specification=job_spec)
self.batch_client.job_schedule.add(setup)
return self.batch_client.job_schedule.get(job_schedule_id=job_configuration.id)
