def submit_job_and_add_task(batch_client, job_id, vm_size, vm_count):
"""Submits a job to the Azure Batch service and adds a simple task.
:param batch_client: The batch client to use.
:type batch_client: `batchserviceclient.BatchServiceClient`
:param str job_id: The id of the job to create.
"""
cloud_service_config = batchmodels.CloudServiceConfiguration(os_family='5')
pool_info = batchmodels.PoolInformation(
auto_pool_specification=batchmodels.AutoPoolSpecification(
auto_pool_id_prefix="HelloWorld",
pool=batchmodels.PoolSpecification(
vm_size=vm_size,
target_dedicated_nodes=vm_count,
cloud_service_configuration=cloud_service_config),
keep_alive=False,
pool_lifetime_option=batchmodels.PoolLifetimeOption.job))
job = batchmodels.JobAddParameter(id=job_id, pool_info=pool_info)
batch_client.job.add(job)
task = batchmodels.TaskAddParameter(
id="HelloWorld",
command_line=common.helpers.wrap_commands_in_shell(
'windows',
['echo Hello world from the Batch Hello world sample!']))
batch_client.task.add(job_id=job.id, task=task)
def create_pool(batch_service_client, pool_id, resource_files):
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str pool_id: An ID for the new pool.
:param list resource_files: A collection of resource files for the pool's
start task.
:param str publisher: Marketplace image publisher
:param str offer: Marketplace image offer
:param str sku: Marketplace image sku
"""
print('Creating pool [{}]...'.format(pool_id))
# Create a new pool of Linux compute nodes using an Azure Virtual Machines
# Marketplace image. For more information about creating pools of Linux
# nodes, see:
# https://azure.microsoft.com/documentation/articles/batch-linux-nodes/
# Specify the commands for the pool's start task. The start task is run
# on each node as it joins the pool, and when it's rebooted or re-imaged.
# We use the start task to prep the node for running our task script.
task_commands = [
# Copy the python_tutorial_task.py script to the "shared" directory
# that all tasks that run on the node have access to.
#'echo %AZ_BATCH_TASK_WORKING_DIR%',
#'echo %AZ_BATCH_NODE_SHARED_DIR%',
'copy %AZ_BATCH_TASK_WORKING_DIR%\* %AZ_BATCH_NODE_SHARED_DIR%',
# Install python with dependencies via PowerShell
'powershell.exe -ExecutionPolicy Unrestricted -File %AZ_BATCH_NODE_SHARED_DIR%\InstallPython35.ps1'
]
new_pool = batch.models.PoolAddParameter(
id=pool_id,
cloud_service_configuration=batchmodels.CloudServiceConfiguration(
os_family="4", target_os_version="*"),
vm_size=_POOL_VM_SIZE,
target_dedicated=_POOL_NODE_COUNT,
start_task=batch.models.StartTask(
command_line=common.helpers.wrap_commands_in_shell(
'windows', task_commands),
run_elevated=True,
wait_for_success=True,
resource_files=resource_files),
)
try:
batch_service_client.pool.add(new_pool)
except batchmodels.batch_error.BatchErrorException as err:
print_batch_exception(err)
raise
def test_batch_jobs(self, **kwargs):
client = self.create_sharedkey_client(**kwargs)
# Test Create Job
auto_pool = models.AutoPoolSpecification(
pool_lifetime_option=models.PoolLifetimeOption.job,
pool=models.PoolSpecification(
vm_size='small',
cloud_service_configuration=models.CloudServiceConfiguration(
os_family='5'
)
)
)
job_prep = models.JobPreparationTask(command_line="cmd /c \"echo hello world\"")
job_release = models.JobReleaseTask(command_line="cmd /c \"echo goodbye world\"")
job_param = models.JobAddParameter(
id=self.get_resource_name('batch_job1_'),
pool_info=models.PoolInformation(
auto_pool_specification=auto_pool
),
job_preparation_task=job_prep,
job_release_task=job_release
)
response = client.job.add(job_param)
self.assertIsNone(response)
# Test Update Job
constraints = models.JobConstraints(max_task_retry_count=3)
options = models.JobUpdateParameter(
priority=500,
constraints=constraints,
pool_info=models.PoolInformation(
auto_pool_specification=auto_pool
)
)
response = client.job.update(job_param.id, options)
self.assertIsNone(response)
# Test Patch Job
options = models.JobPatchParameter(priority=900)
response = client.job.patch(job_param.id, options)
self.assertIsNone(response)
job = client.job.get(job_param.id)
self.assertIsInstance(job, models.CloudJob)
self.assertEqual(job.id, job_param.id)
self.assertEqual(job.constraints.max_task_retry_count, 3)
self.assertEqual(job.priority, 900)
# Test Create Job with Auto Complete
job_auto_param = models.JobAddParameter(
id=self.get_resource_name('batch_job2_'),
on_all_tasks_complete=models.OnAllTasksComplete.terminate_job,
on_task_failure=models.OnTaskFailure.perform_exit_options_job_action,
pool_info=models.PoolInformation(
auto_pool_specification=auto_pool
)
)
response = client.job.add(job_auto_param)
self.assertIsNone(response)
job = client.job.get(job_auto_param.id)
self.assertIsInstance(job, models.CloudJob)
self.assertEqual(job.on_all_tasks_complete, models.OnAllTasksComplete.terminate_job)
self.assertEqual(job.on_task_failure, models.OnTaskFailure.perform_exit_options_job_action)
# Test List Jobs
jobs = client.job.list()
self.assertIsInstance(jobs, models.CloudJobPaged)
self.assertEqual(len(list(jobs)), 2)
# Test Disable Job
response = client.job.disable(job_param.id, models.DisableJobOption.requeue)
self.assertIsNone(response)
# Test Enable Job
response = client.job.enable(job_param.id)
self.assertIsNone(response)
# Prep and release task status
task_status = client.job.list_preparation_and_release_task_status(job_param.id)
self.assertIsInstance(task_status, models.JobPreparationAndReleaseTaskExecutionInformationPaged)
self.assertEqual(list(task_status), [])
# Test Terminate Job
response = client.job.terminate(job_param.id)
self.assertIsNone(response)
# Test Delete Job
response = client.job.delete(job_auto_param.id)
self.assertIsNone(response)
# Test Job Lifetime Statistics
stats = client.job.get_all_lifetime_statistics()
self.assertIsInstance(stats, models.JobStatistics)
self.assertEqual(stats.num_succeeded_tasks, 0)
self.assertEqual(stats.num_failed_tasks, 0)
def test_batch_update_pools(self, **kwargs):
client = self.create_sharedkey_client(**kwargs)
# Test Create Paas Pool
test_paas_pool = models.PoolAddParameter(
id=self.get_resource_name('batch_paas_'),
vm_size='small',
cloud_service_configuration=models.CloudServiceConfiguration(
os_family='5'
),
start_task=models.StartTask(
command_line="cmd.exe /c \"echo hello world\"",
resource_files=[models.ResourceFile('https://blobsource.com', 'filename.txt')],
environment_settings=[models.EnvironmentSetting('ENV_VAR', 'env_value')],
user_identity=models.UserIdentity(
auto_user=models.AutoUserSpecification(
elevation_level=models.ElevationLevel.admin
)
)
)
)
response = client.pool.add(test_paas_pool)
self.assertIsNone(response)
# Test Upgrade Pool OS
self.assertBatchError(
"PoolVersionEqualsUpgradeVersion",
client.pool.upgrade_os,
test_paas_pool.id,
"*"
)
# Test Update Pool Parameters
params = models.PoolUpdatePropertiesParameter([], [], [models.MetadataItem('foo', 'bar')])
response = client.pool.update_properties(test_paas_pool.id, params)
self.assertIsNone(response)
# Test Patch Pool Parameters
params = models.PoolPatchParameter(metadata=[models.MetadataItem('foo2', 'bar2')])
response = client.pool.patch(test_paas_pool.id, params)
self.assertIsNone(response)
# Test Pool Exists
response = client.pool.exists(test_paas_pool.id)
self.assertTrue(response)
# Test Get Pool
pool = client.pool.get(test_paas_pool.id)
self.assertIsInstance(pool, models.CloudPool)
self.assertEqual(pool.id, test_paas_pool.id)
self.assertEqual(pool.state, models.PoolState.active)
self.assertEqual(pool.allocation_state, models.AllocationState.steady)
self.assertEqual(pool.cloud_service_configuration.os_family, '5')
self.assertEqual(pool.vm_size, 'small')
self.assertIsNone(pool.start_task)
self.assertEqual(pool.metadata[0].name, 'foo2')
self.assertEqual(pool.metadata[0].value, 'bar2')
# Test Get Pool with OData Clauses
options = models.PoolGetOptions(select='id,state', expand='stats')
pool = client.pool.get(test_paas_pool.id, options)
self.assertIsInstance(pool, models.CloudPool)
self.assertEqual(pool.id, test_paas_pool.id)
self.assertEqual(pool.state, models.PoolState.active)
self.assertIsNone(pool.allocation_state)
self.assertIsNone(pool.vm_size)
# Test Delete Pool
response = client.pool.delete(test_paas_pool.id)
self.assertIsNone(response)
def configure_pool( # pylint: disable=too-many-arguments
self,
pool_id: str,
vm_size: Optional[str] = None,
vm_publisher: Optional[str] = None,
vm_offer: Optional[str] = None,
sku_starts_with: Optional[str] = None,
vm_sku: Optional[str] = None,
vm_version: Optional[str] = None,
vm_node_agent_sku_id: Optional[str] = None,
os_family: Optional[str] = None,
os_version: Optional[str] = None,
display_name: Optional[str] = None,
target_dedicated_nodes: Optional[int] = None,
use_latest_image_and_sku: bool = False,
**kwargs,
) -> PoolAddParameter:
"""
Configures a pool
:param pool_id: A string that uniquely identifies the Pool within the Account
:type pool_id: str
:param vm_size: The size of virtual machines in the Pool.
:type vm_size: str
:param display_name: The display name for the Pool
:type display_name: str
:param target_dedicated_nodes: The desired number of dedicated Compute Nodes in the Pool.
:type target_dedicated_nodes: Optional[int]
:param use_latest_image_and_sku: Whether to use the latest verified vm image and sku
:type use_latest_image_and_sku: bool
:param vm_publisher: The publisher of the Azure Virtual Machines Marketplace Image.
For example, Canonical or MicrosoftWindowsServer.
:type vm_publisher: Optional[str]
:param vm_offer: The offer type of the Azure Virtual Machines Marketplace Image.
For example, UbuntuServer or WindowsServer.
:type vm_offer: Optional[str]
:param sku_starts_with: The start name of the sku to search
:type sku_starts_with: Optional[str]
:param vm_sku: The name of the virtual machine sku to use
:type vm_sku: Optional[str]
:param vm_version: The version of the virtual machine
:param vm_version: str
:param vm_node_agent_sku_id: The node agent sku id of the virtual machine
:type vm_node_agent_sku_id: Optional[str]
:param os_family: The Azure Guest OS family to be installed on the virtual machines in the Pool.
:type os_family: Optional[str]
:param os_version: The OS family version
:type os_version: Optional[str]
"""
if use_latest_image_and_sku:
self.log.info('Using latest verified virtual machine image with node agent sku')
sku_to_use, image_ref_to_use = self._get_latest_verified_image_vm_and_sku(
publisher=vm_publisher, offer=vm_offer, sku_starts_with=sku_starts_with
)
pool = batch_models.PoolAddParameter(
id=pool_id,
vm_size=vm_size,
display_name=display_name,
virtual_machine_configuration=batch_models.VirtualMachineConfiguration(
image_reference=image_ref_to_use, node_agent_sku_id=sku_to_use
),
target_dedicated_nodes=target_dedicated_nodes,
**kwargs,
)
elif os_family:
self.log.info(
'Using cloud service configuration to create pool, virtual machine configuration ignored'
)
pool = batch_models.PoolAddParameter(
id=pool_id,
vm_size=vm_size,
display_name=display_name,
cloud_service_configuration=batch_models.CloudServiceConfiguration(
os_family=os_family, os_version=os_version
),
target_dedicated_nodes=target_dedicated_nodes,
**kwargs,
)
else:
self.log.info('Using virtual machine configuration to create a pool')
pool = batch_models.PoolAddParameter(
id=pool_id,
vm_size=vm_size,
display_name=display_name,
virtual_machine_configuration=batch_models.VirtualMachineConfiguration(
image_reference=batch_models.ImageReference(
publisher=vm_publisher,
offer=vm_offer,
sku=vm_sku,
version=vm_version,
),
node_agent_sku_id=vm_node_agent_sku_id,
),
target_dedicated_nodes=target_dedicated_nodes,
**kwargs,
)
return pool
def create_job_schedule(batch_client, job_schedule_id, vm_size, vm_count,
block_blob_client):
"""Creates an Azure Batch pool and job schedule with the specified ids.
:param batch_client: The batch client to use.
:type batch_client: `batchserviceclient.BatchServiceClient`
:param str job_schedule_id: The id of the job schedule to create
:param str vm_size: vm size (sku)
:param int vm_count: number of vms to allocate
:param block_blob_client: The storage block blob client to use.
:type block_blob_client: `azure.storage.blob.BlockBlobService`
"""
cloud_service_config = batchmodels.CloudServiceConfiguration(os_family='6')
user_id = batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
elevation_level=_USER_ELEVATION_LEVEL))
python_download = batchmodels.ResourceFile(http_url=_PYTHON_DOWNLOAD,
file_path='python373.exe')
pool_info = batchmodels.PoolInformation(
auto_pool_specification=batchmodels.AutoPoolSpecification(
auto_pool_id_prefix="JobScheduler",
pool=batchmodels.PoolSpecification(
vm_size=vm_size,
target_dedicated_nodes=vm_count,
cloud_service_configuration=cloud_service_config,
start_task=batchmodels.StartTask(
command_line=common.helpers.wrap_commands_in_shell(
'windows', ['{}'.format(_PYTHON_INSTALL)]),
resource_files=[python_download],
wait_for_success=True,
user_identity=user_id)),
keep_alive=False,
pool_lifetime_option=batchmodels.PoolLifetimeOption.job))
sas_url = common.helpers.upload_blob_and_create_sas(
block_blob_client, _CONTAINER_NAME, _SIMPLE_TASK_NAME,
_SIMPLE_TASK_PATH,
datetime.datetime.utcnow() + datetime.timedelta(minutes=30))
job_spec = batchmodels.JobSpecification(
pool_info=pool_info,
# Terminate job once all tasks under it are complete to allow for a new
# job to be created under the schedule
on_all_tasks_complete=batchmodels.OnAllTasksComplete.terminate_job,
job_manager_task=batchmodels.JobManagerTask(
id="JobManagerTask",
command_line=common.helpers.wrap_commands_in_shell(
'windows', ['python {}'.format(_SIMPLE_TASK_NAME)]),
resource_files=[
batchmodels.ResourceFile(file_path=_SIMPLE_TASK_NAME,
http_url=sas_url)
]))
do_not_run_after = datetime.datetime.utcnow() \
+ datetime.timedelta(minutes=30)
schedule = batchmodels.Schedule(
do_not_run_after=do_not_run_after,
recurrence_interval=datetime.timedelta(minutes=10))
scheduled_job = batchmodels.JobScheduleAddParameter(
id=job_schedule_id, schedule=schedule, job_specification=job_spec)
batch_client.job_schedule.add(cloud_job_schedule=scheduled_job)
def get_pool(self, pool_id, vm_size="standard_d15_v2", node_count=0):
'''
Creates an Azure Batch pool.
:param pool_id: The pool_id of the pool to create
:type pool_id: string
:param vm_size: the type of compute nodes in the pool. (Defaults to 'standard_d15_v2')
:type vm_size: string
:param node_count: The number of compute nodes to initially create in the pool. Defaults to 0.
:type node_count: number
:rtype: string
:return: the pool_id of the create pool
'''
batch_client = self.batch_client()
pool_list = list(batch_client.pool.list())
for pool in pool_list:
if pool.id == pool_id: #We have a pool with this pool_id, is it busy?
node_list = list(batch_client.compute_node.list(pool.id))
for node in node_list:
if node.running_tasks_count > 0:
logging.info(
"pool '{0}' exists and is busy".format(pool_id))
break
return pool.id
logging.info(
"pool '{0}' does not exist and will be created".format(pool_id))
user_identity = batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
elevation_level='admin'))
start_task = batchmodels.StartTask(
command_line="cmd /c %AZ_BATCH_APP_PACKAGE_STARTUP%\startup.bat",
user_identity=user_identity,
wait_for_success=True)
new_pool = batchmodels.PoolAddParameter(id=pool_id,
vm_size=vm_size,
start_task=start_task)
new_pool.target_dedicated = node_count
new_pool.max_tasks_per_node = 1
cloud_service_configuration = batchmodels.CloudServiceConfiguration(
os_family=4)
new_pool.cloud_service_configuration = cloud_service_configuration
new_pool.application_package_references = [
batchmodels.ApplicationPackageReference("anaconda2"),
batchmodels.ApplicationPackageReference("startup")
]
#add the vnet
#The ARM resource identifier of the virtual network subnet which the compute nodes of the pool will join. The virtual
#network must be in the same region and subscription as the Azure Batch
#account. This property can only be specified for pools created with a
#cloudServiceConfiguration.
# value u'Property subnetId should be of the form /subscriptions/{0}/resourceGroups/{1}/providers/{2}/virtualNetworks/{3}/subnets/{4}' unicode
new_pool.network_configuration = batchmodels.NetworkConfiguration(
self.vnet + "/subnets/default")
try:
batch_client.pool.add(new_pool)
except Exception, e:
print e
