def create_pool_and_wait_for_node(batch_client, pool_id, vm_size, vm_count,
sha1_cert_tp):
"""Creates an Azure Batch pool with the specified id.
:param batch_client: The batch client to use.
:type batch_client: `batchserviceclient.BatchServiceClient`
:param str pool_id: The id of the pool to create.
:param str vm_size: vm size (sku)
:param int vm_count: number of vms to allocate
:param str sha1_cert_tp: sha1 cert thumbprint for cert ref
"""
# pick the latest supported 16.04 sku for UbuntuServer
sku_to_use, image_ref_to_use = \
common.helpers.select_latest_verified_vm_image_with_node_agent_sku(
batch_client, 'Canonical', 'UbuntuServer', '16.04')
# create start task commands
# 1. update repository
# 2. install blobxfer pre-requisites
# 3. pip install blobxfer python script
start_task_commands = [
'apt-get update',
'apt-get install -y build-essential libssl-dev libffi-dev ' +
'libpython-dev python-dev python-pip', 'pip install --upgrade blobxfer'
]
user = batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)
# create pool with start task and cert ref with visibility of task
pool = batchmodels.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=image_ref_to_use, node_agent_sku_id=sku_to_use),
vm_size=vm_size,
target_dedicated=vm_count,
start_task=batchmodels.StartTask(
command_line=common.helpers.wrap_commands_in_shell(
'linux', start_task_commands),
user_identity=batchmodels.UserIdentity(auto_user=user),
wait_for_success=True),
certificate_references=[
batchmodels.CertificateReference(
sha1_cert_tp,
'sha1',
visibility=[batchmodels.CertificateVisibility.task])
],
)
common.helpers.create_pool_if_not_exist(batch_client, pool)
def create_pool_and_wait_for_node(batch_client, pool_id, vm_size, vm_count,
sha1_cert_tp):
"""Creates an Azure Batch pool with the specified id.
:param batch_client: The batch client to use.
:type batch_client: `batchserviceclient.BatchServiceClient`
:param str pool_id: The id of the pool to create.
:param str vm_size: vm size (sku)
:param int vm_count: number of vms to allocate
:param str sha1_cert_tp: sha1 cert thumbprint for cert ref
"""
# pick the latest supported 16.04 sku for UbuntuServer
sku_to_use, image_ref_to_use = \
common.helpers.select_latest_verified_vm_image_with_node_agent_sku(
batch_client, 'Canonical', 'UbuntuServer', '16.04')
# create start task commands
# 1. update repository
# 2. install blobxfer pre-requisites
# 3. pip install blobxfer python script
start_task_commands = [
'apt-get update',
'apt-get install -y build-essential libssl-dev libffi-dev ' +
'libpython-dev python-dev python-pip', 'pip install --upgrade blobxfer'
]
user = batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)
# create pool with start task and cert ref with visibility of task
pool = batchmodels.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=image_ref_to_use, node_agent_sku_id=sku_to_use),
vm_size=vm_size,
target_dedicated_nodes=vm_count,
start_task=batchmodels.StartTask(
command_line=common.helpers.wrap_commands_in_shell(
'linux', start_task_commands),
user_identity=batchmodels.UserIdentity(auto_user=user),
wait_for_success=True),
certificate_references=[
batchmodels.CertificateReference(
thumbprint=sha1_cert_tp,
thumbprint_algorithm='sha1',
visibility=[batchmodels.CertificateVisibility.task])
],
)
common.helpers.create_pool_if_not_exist(batch_client, pool)
# because we want all nodes to be available before any tasks are assigned
# to the pool, here we will wait for all compute nodes to reach idle
nodes = common.helpers.wait_for_all_nodes_state(
batch_client, pool,
frozenset((batchmodels.ComputeNodeState.start_task_failed,
batchmodels.ComputeNodeState.unusable,
batchmodels.ComputeNodeState.idle)))
# ensure all node are idle
if any(node.state != batchmodels.ComputeNodeState.idle for node in nodes):
raise RuntimeError(
'node(s) of pool {} not in idle state'.format(pool_id))
def create_pool(self,
pool_id,
vm_size,
target_dedicated,
target_low_priority,
batch_image_spec,
starttask_cmd,
starttask_url,
starttask_script,
sp_cert_thumb,
app_licenses=None,
disable_remote_access=True,
app_pkgs=None,
subnet_id=None,
app_insights_app_key=None,
app_insights_instrumentation_key=None):
pool = batchmodels.PoolAddParameter(
id=pool_id,
display_name=pool_id,
vm_size=vm_size,
target_dedicated_nodes=target_dedicated,
target_low_priority_nodes=target_low_priority,
virtual_machine_configuration=batch_image_spec.
get_virtual_machine_configuration(),
application_package_references=app_pkgs,
certificate_references=[
batchmodels.CertificateReference(sp_cert_thumb, 'sha1')
])
if app_licenses:
pool.application_licenses = app_licenses
pool.start_task = batchmodels.StartTask(
command_line=starttask_cmd,
max_task_retry_count=3,
user_identity=batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)),
wait_for_success=True,
resource_files=[
batchmodels.ResourceFile(starttask_url, starttask_script)
])
if app_insights_app_key and app_insights_instrumentation_key:
pool.start_task.environment_settings = [
batchmodels.EnvironmentSetting('APP_INSIGHTS_APP_ID',
app_insights_app_key),
batchmodels.EnvironmentSetting(
'APP_INSIGHTS_INSTRUMENTATION_KEY',
app_insights_instrumentation_key)
]
if subnet_id:
pool.network_configuration = batchmodels.NetworkConfiguration(
subnet_id=subnet_id)
if disable_remote_access:
if pool.network_configuration is None:
pool.network_configuration = batchmodels.NetworkConfiguration()
endpoint_config = batchmodels.PoolEndpointConfiguration(
inbound_nat_pools=[
batchmodels.InboundNATPool(
'DisableRDP',
batchmodels.InboundEndpointProtocol.tcp,
3389,
60000,
60099,
network_security_group_rules=[
batchmodels.NetworkSecurityGroupRule(
150, batchmodels.
NetworkSecurityGroupRuleAccess.deny, '*')
]),
batchmodels.InboundNATPool(
'DisableSSH',
batchmodels.InboundEndpointProtocol.tcp,
22,
61000,
61099,
network_security_group_rules=[
batchmodels.NetworkSecurityGroupRule(
151, batchmodels.
NetworkSecurityGroupRuleAccess.deny, '*')
])
])
pool.network_configuration.endpoint_configuration = endpoint_config
try:
client = self._get_batch_client()
client.pool.add(pool)
except batchmodels.BatchErrorException as be:
if be.error:
print('Error creating pool, code={}, message={}'.format(
be.error.code, be.error.message))
if be.error.values:
for e in be.error.values:
print('Key={}, Value={}'.format(e.key, e.value))
raise