def test_batch_create_pool_with_blobfuse_mount(self, **kwargs):
client = self.create_sharedkey_client(**kwargs)
# Test Create Iaas Pool
test_iaas_pool = models.PoolAddParameter(
id=self.get_resource_name('batch_iaas_'),
vm_size='Standard_A1',
virtual_machine_configuration=models.VirtualMachineConfiguration(
image_reference=models.ImageReference(
publisher='MicrosoftWindowsServer',
offer='WindowsServer',
sku='2016-Datacenter-smalldisk'
),
node_agent_sku_id='batch.node.windows amd64',
windows_configuration=models.WindowsConfiguration(enable_automatic_updates=True)),
task_scheduling_policy=models.TaskSchedulingPolicy(node_fill_type=models.ComputeNodeFillType.pack),
mount_configuration=[models.MountConfiguration(
azure_blob_file_system_configuration=models.AzureBlobFileSystemConfiguration(
account_name='test',
container_name='https://test.blob.core.windows.net:443/test-container',
relative_mount_path='foo',
account_key='fake_account_key'
)
)]
)
response = client.pool.add(test_iaas_pool)
self.assertIsNone(response)
mount_pool = client.pool.get(test_iaas_pool.id)
self.assertIsNotNone(mount_pool.mount_configuration)
self.assertEqual(len(mount_pool.mount_configuration), 1)
self.assertIsNotNone(mount_pool.mount_configuration[0].azure_blob_file_system_configuration)
self.assertIsNone(mount_pool.mount_configuration[0].nfs_mount_configuration)
def create_pool(self, dedicated_node_count, low_priority_node_count,
num_cores_per_node, pool_vm_size, publisher, offer, sku,
start_task, resource_files):
"""Starts pool creation on Azure
Args:
dedicated_node_count: Number of dedicated nodes in the pool
low_priority_node_count: number of low priority nodes in pool
num_cores_per_node: Number of cores per node; depends on the pool_vm_size
pool_vm_size: The type of VM to run on each node;
see https://docs.microsoft.com/en-us/azure/cloud-services/cloud-services-sizes-specs
publisher: VM image publisher (ex Canonical)
offer: VM image offer (ex UbuntuServer)
sku: VM image sku (ex 16.04)
start_task: command to run when each node joins the bool formatted as a string
resource_files: list of file references (of type azure.batch.models.ResourceFile)
from Azure Storage to download to each node
Returns:
True if creation could be started successfully. False if pool already exists.
Throws error if something else went wrong.
"""
sku_to_use, image_ref_to_use = self._select_latest_verified_vm_image_with_node_agent_sku(
publisher, offer, sku)
new_pool = batch_models.PoolAddParameter(
id=self.pool_id,
virtual_machine_configuration=batch_models.
VirtualMachineConfiguration(image_reference=image_ref_to_use,
node_agent_sku_id=sku_to_use),
vm_size=pool_vm_size,
target_dedicated_nodes=dedicated_node_count,
target_low_priority_nodes=low_priority_node_count,
start_task=self._create_start_task(start_task, resource_files),
max_tasks_per_node=num_cores_per_node,
task_scheduling_policy=batch_models.TaskSchedulingPolicy(
batch_models.ComputeNodeFillType.spread))
try:
logging.info("Attempting to create pool [{}]...".format(
self.pool_id))
self.batch_client.pool.add(new_pool)
logging.info("Pool [{}] created successfully...".format(
self.pool_id))
return True
except batch_models.BatchErrorException as e:
if e.error.code == "PoolExists":
logging.info("Pool [{}] already exists".format(self.pool_id))
return False
else:
logging.exception(
"Unknown error occurred while trying to create pool [{}]".
format(self.pool_id))
raise
def test_batch_create_pools(self, **kwargs):
client = self.create_sharedkey_client(**kwargs)
# Test List Node Agent SKUs
response = client.account.list_node_agent_skus()
response = list(response)
self.assertTrue(len(response) > 1)
self.assertEqual(response[-1].id, "batch.node.windows amd64")
self.assertEqual(response[-1].os_type.value, "windows")
self.assertTrue(len(response[-1].verified_image_references) > 1)
# Test Create Iaas Pool
users = [
models.UserAccount('test-user-1', 'kt#_gahr!@aGERDXA'),
models.UserAccount('test-user-2', 'kt#_gahr!@aGERDXA', models.ElevationLevel.admin)
]
test_iaas_pool = models.PoolAddParameter(
id=self.get_resource_name('batch_iaas_'),
vm_size='Standard_A1',
virtual_machine_configuration=models.VirtualMachineConfiguration(
image_reference=models.ImageReference(
publisher='MicrosoftWindowsServer',
offer='WindowsServer',
sku='2016-Datacenter-smalldisk'
),
node_agent_sku_id='batch.node.windows amd64',
windows_configuration=models.WindowsConfiguration(True)),
task_scheduling_policy=models.TaskSchedulingPolicy(models.ComputeNodeFillType.pack),
user_accounts=users
)
response = client.pool.add(test_iaas_pool)
self.assertIsNone(response)
# Test list pool node counnt
counts = list(client.account.list_pool_node_counts())
self.assertIsNotNone(counts)
self.assertEqual(len(counts), 1)
self.assertEqual(counts[0].pool_id, test_iaas_pool.id)
self.assertIsNotNone(counts[0].dedicated)
self.assertEqual(counts[0].dedicated.total, 0)
self.assertEqual(counts[0].low_priority.total, 0)
# Test Create Pool with Network Configuration
network_config = models.NetworkConfiguration('/subscriptions/00000000-0000-0000-0000-000000000000'
'/resourceGroups/test'
'/providers/Microsoft.Network'
'/virtualNetworks/vnet1'
'/subnets/subnet1')
test_network_pool = models.PoolAddParameter(
id=self.get_resource_name('batch_network_'),
vm_size='Standard_A1',
network_configuration=network_config,
virtual_machine_configuration=models.VirtualMachineConfiguration(
image_reference=models.ImageReference(
publisher='Canonical',
offer='UbuntuServer',
sku='16.04-LTS'
),
node_agent_sku_id='batch.node.ubuntu 16.04')
)
self.assertBatchError('InvalidPropertyValue', client.pool.add, test_network_pool, models.PoolAddOptions(timeout=45))
# Test Create Pool with Custom Image
test_image_pool = models.PoolAddParameter(
id=self.get_resource_name('batch_image_'),
vm_size='Standard_A1',
virtual_machine_configuration=models.VirtualMachineConfiguration(
image_reference=models.ImageReference(
virtual_machine_image_id="/subscriptions/00000000-0000-0000-0000-000000000000"
"/resourceGroups/test"
"/providers/Microsoft.Compute"
"/images/FakeImage"
),
node_agent_sku_id='batch.node.ubuntu 16.04'
)
)
self.assertBatchError('InvalidPropertyValue', client.pool.add, test_image_pool, models.PoolAddOptions(timeout=45))
# Test Create Pool with OSDisk
os_disk = models.OSDisk(caching=models.CachingType.read_write)
test_osdisk_pool = models.PoolAddParameter(
id=self.get_resource_name('batch_osdisk_'),
vm_size='Standard_A1',
virtual_machine_configuration=models.VirtualMachineConfiguration(
image_reference=models.ImageReference(
publisher='Canonical',
offer='UbuntuServer',
sku='16.04-LTS'
),
node_agent_sku_id='batch.node.ubuntu 16.04',
os_disk=os_disk)
)
response = client.pool.add(test_osdisk_pool)
self.assertIsNone(response)
osdisk_pool = client.pool.get(test_osdisk_pool.id)
self.assertEqual(osdisk_pool.virtual_machine_configuration.os_disk.caching, os_disk.caching)
# Test Create Pool with Data Disk
data_disk = models.DataDisk(lun=1, disk_size_gb=50)
test_disk_pool = models.PoolAddParameter(
id=self.get_resource_name('batch_disk_'),
vm_size='Standard_A1',
virtual_machine_configuration=models.VirtualMachineConfiguration(
image_reference=models.ImageReference(
publisher='Canonical',
offer='UbuntuServer',
sku='16.04-LTS'
),
node_agent_sku_id='batch.node.ubuntu 16.04',
data_disks=[data_disk])
)
response = client.pool.add(test_disk_pool)
self.assertIsNone(response)
disk_pool = client.pool.get(test_disk_pool.id)
self.assertEqual(disk_pool.virtual_machine_configuration.data_disks[0].lun, 1)
self.assertEqual(disk_pool.virtual_machine_configuration.data_disks[0].disk_size_gb, 50)
# Test Create Pool with Application Licenses
test_app_pool = models.PoolAddParameter(
id=self.get_resource_name('batch_app_'),
vm_size='Standard_A1',
application_licenses=["maya"],
virtual_machine_configuration=models.VirtualMachineConfiguration(
image_reference=models.ImageReference(
publisher='Canonical',
offer='UbuntuServer',
sku='16.04-LTS'
),
node_agent_sku_id='batch.node.ubuntu 16.04',
data_disks=[data_disk])
)
response = client.pool.add(test_app_pool)
self.assertIsNone(response)
app_pool = client.pool.get(test_app_pool.id)
self.assertEqual(app_pool.application_licenses[0], "maya")
# Test List Pools without Filters
pools = list(client.pool.list())
self.assertTrue(len(pools) > 1)
# Test List Pools with Maximum
options = models.PoolListOptions(max_results=1)
pools = client.pool.list(options)
pools.next()
self.assertEqual(len(pools.current_page), 1)
# Test List Pools with Filter
options = models.PoolListOptions(
filter='startswith(id,\'batch_app_\')',
select='id,state',
expand='stats')
pools = list(client.pool.list(options))
self.assertEqual(len(pools), 1)
def create_pool(batch_service_client, pool_id, scale_interval,
auto_scale_eval_interval, dedicated_node_count):
"""Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str pool_id: An ID for the new pool.
:param str scale_interval: as in auto scale formula.
:param timedelta auto_scale_eval_interval: time that specify the auto scale evaluation interval
:param int dedicated_node_count: number of dedicated nodes in pool."""
# TODO: monitor nodes
# check core quota
cores_limit = _BATCH_CORE_QUOTA
cores = subprocess.run(['./countCoresUsed.sh ' + _POOL_VM_SIZE],
shell=True,
capture_output=True).stdout
cores = cores.decode().split(',')
if _MAX_TASK_PER_NODE > int(cores[1]):
print('Max. task/node is more that cores/node! Exiting...')
exit(1)
if dedicated_node_count * int(cores[1]) + int(cores[0]) > cores_limit:
dedicated_node_count = floor(
(cores_limit - int(cores[0])) / int(cores[1]))
print('Core limit {} reached!'.format(cores_limit))
print('Creating pool [{}] with [{}] dedicated nodes'.format(
pool_id, dedicated_node_count))
print('VM size is [{}], vCPUs is [{}] each, max. task(s)/node is [{}]'.
format(_POOL_VM_SIZE, int(cores[1]), _MAX_TASK_PER_NODE))
if int(cores[0]) > 0:
print('Cores currently in use from other pool(s) is [{}]'.format(
int(cores[0])))
# Create a new pool of Linux compute nodes using an Azure Virtual Machines Marketplace or customized image.
# For more information about creating pools of Linux nodes, see:
# https://azure.microsoft.com/documentation/articles/batch-linux-nodes/
new_pool = batchmodels.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=batchmodels.ImageReference(publisher="Canonical",
offer="UbuntuServer",
sku="16.04-LTS",
version="latest"),
node_agent_sku_id='batch.node.ubuntu 16.04'),
vm_size=_POOL_VM_SIZE,
target_dedicated_nodes=dedicated_node_count,
max_tasks_per_node=_MAX_TASK_PER_NODE,
enable_inter_node_communication=False,
resize_timeout=datetime.timedelta(minutes=10),
task_scheduling_policy=batchmodels.TaskSchedulingPolicy(
node_fill_type='spread'),
# Configure the start task for the pool
# https://docs.microsoft.com/en-us/python/api/azure-batch/azure.batch.models.starttask?view=azure-python
start_task=batchmodels.StartTask(
command_line='sleep 1',
user_identity=batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
elevation_level=batchmodels.ElevationLevel.admin,
scope=batchmodels.AutoUserScope.pool)),
# environment_settings=[batchmodels.EnvironmentSetting(
# name='LD_LIBRARY_PATH', value='/opt/mcr18b/v95/runtime/glnxa64:'
# '/opt/mcr18b/v95/bin/glnxa64:/opt/mcr18b/v95/sys/os/glnxa64')],
wait_for_success=1)
# resource_files=[
# batchmodels.ResourceFile(file_path=STARTTASK_RESOURCE_FILE, blob_source=SAS_URL)
# ])
)
batch_service_client.pool.add(new_pool)
while True:
new_pool = batch_service_client.pool.get(pool_id)
if new_pool.allocation_state != 'steady':
print(time.strftime('[%H:%M:%S %x]') +
' Waiting pool to be steady',
end='\r')
time.sleep(1)
else:
break
batch_service_client.pool.enable_auto_scale(
pool_id,
auto_scale_formula=
('$NodeDeallocationOption=taskcompletion;$tasks=($PendingTasks.'
'GetSamplePercent(TimeInterval_{})<70) ? max(0,$PendingTasks.GetSample(1)) : '
'max($PendingTasks.GetSample(1),avg($PendingTasks.GetSample(TimeInterval_{})));'
'$targetVMs = max($tasks/{}, max($RunningTasks.GetSample(TimeInterval_{})/{}));'
'$TargetDedicated = min({}, $targetVMs)'.format(
scale_interval, scale_interval, _MAX_TASK_PER_NODE,
scale_interval, _MAX_TASK_PER_NODE, dedicated_node_count)),
auto_scale_evaluation_interval=auto_scale_eval_interval)
print(
time.strftime('[%H:%M:%S %x]') + ' Pool steady and autoscale enabled')
def test_batch_create_pools(self, **kwargs):
client = self.create_sharedkey_client(**kwargs)
# Test List Node Agent SKUs
response = client.account.list_supported_images()
response = list(response)
self.assertTrue(len(response) > 1)
self.assertIsNotNone(response[-1].image_reference)
# Test Create Iaas Pool
users = [
models.UserAccount(name='test-user-1', password='******'),
models.UserAccount(name='test-user-2', password='******', elevation_level=models.ElevationLevel.admin)
]
test_iaas_pool = models.PoolAddParameter(
id=self.get_resource_name('batch_iaas_'),
vm_size=DEFAULT_VM_SIZE,
virtual_machine_configuration=models.VirtualMachineConfiguration(
image_reference=models.ImageReference(
publisher='MicrosoftWindowsServer',
offer='WindowsServer',
sku='2016-Datacenter-smalldisk'
),
node_agent_sku_id='batch.node.windows amd64',
windows_configuration=models.WindowsConfiguration(enable_automatic_updates=True)),
task_scheduling_policy=models.TaskSchedulingPolicy(node_fill_type=models.ComputeNodeFillType.pack),
user_accounts=users
)
response = client.pool.add(test_iaas_pool)
self.assertIsNone(response)
# Test list pool node counnt
counts = list(client.account.list_pool_node_counts())
self.assertIsNotNone(counts)
self.assertEqual(len(counts), 1)
self.assertEqual(counts[0].pool_id, test_iaas_pool.id)
self.assertIsNotNone(counts[0].dedicated)
self.assertEqual(counts[0].dedicated.total, 0)
self.assertEqual(counts[0].dedicated.leaving_pool, 0)
self.assertEqual(counts[0].low_priority.total, 0)
# Test Create Pool with Network Configuration
#TODO Public IP tests
network_config = models.NetworkConfiguration(subnet_id='/subscriptions/00000000-0000-0000-0000-000000000000'
'/resourceGroups/test'
'/providers/Microsoft.Network'
'/virtualNetworks/vnet1'
'/subnets/subnet1')
test_network_pool = models.PoolAddParameter(
id=self.get_resource_name('batch_network_'),
vm_size=DEFAULT_VM_SIZE,
network_configuration=network_config,
virtual_machine_configuration=models.VirtualMachineConfiguration(
image_reference=models.ImageReference(
publisher='Canonical',
offer='UbuntuServer',
sku='18.04-LTS'
),
node_agent_sku_id='batch.node.ubuntu 18.04')
)
self.assertBatchError('InvalidPropertyValue', client.pool.add, test_network_pool, models.PoolAddOptions(timeout=45))
test_image_pool = models.PoolAddParameter(
id=self.get_resource_name('batch_image_'),
vm_size=DEFAULT_VM_SIZE,
virtual_machine_configuration=models.VirtualMachineConfiguration(
image_reference=models.ImageReference(
virtual_machine_image_id="/subscriptions/00000000-0000-0000-0000-000000000000"
"/resourceGroups/test"
"/providers/Microsoft.Compute"
"/gallery/FakeGallery"
"/images/FakeImage"
"/versions/version"
),
node_agent_sku_id='batch.node.ubuntu 18.04'
)
)
self.assertBatchError('InvalidPropertyValue', client.pool.add, test_image_pool, models.PoolAddOptions(timeout=45))
# Test Create Pool with Data Disk
data_disk = models.DataDisk(lun=1, disk_size_gb=50)
test_disk_pool = models.PoolAddParameter(
id=self.get_resource_name('batch_disk_'),
vm_size=DEFAULT_VM_SIZE,
virtual_machine_configuration=models.VirtualMachineConfiguration(
image_reference=models.ImageReference(
publisher='Canonical',
offer='UbuntuServer',
sku='18.04-LTS'
),
node_agent_sku_id='batch.node.ubuntu 18.04',
data_disks=[data_disk])
)
response = client.pool.add(test_disk_pool)
self.assertIsNone(response)
disk_pool = client.pool.get(test_disk_pool.id)
self.assertEqual(disk_pool.virtual_machine_configuration.data_disks[0].lun, 1)
self.assertEqual(disk_pool.virtual_machine_configuration.data_disks[0].disk_size_gb, 50)
# Test Create Pool with Application Licenses
test_app_pool = models.PoolAddParameter(
id=self.get_resource_name('batch_app_'),
vm_size=DEFAULT_VM_SIZE,
application_licenses=["maya"],
virtual_machine_configuration=models.VirtualMachineConfiguration(
image_reference=models.ImageReference(
publisher='Canonical',
offer='UbuntuServer',
sku='18.04-LTS'
),
node_agent_sku_id='batch.node.ubuntu 18.04',
data_disks=[data_disk])
)
response = client.pool.add(test_app_pool)
self.assertIsNone(response)
app_pool = client.pool.get(test_app_pool.id)
self.assertEqual(app_pool.application_licenses[0], "maya")
# Test Create Pool with Azure Disk Encryption
test_ade_pool = models.PoolAddParameter(
id=self.get_resource_name('batch_ade_'),
vm_size=DEFAULT_VM_SIZE,
virtual_machine_configuration=models.VirtualMachineConfiguration(
image_reference=models.ImageReference(
publisher='Canonical',
offer='UbuntuServer',
sku='18.04-LTS'
),
disk_encryption_configuration=models.DiskEncryptionConfiguration(
targets=[models.DiskEncryptionTarget.temporary_disk]
),
node_agent_sku_id='batch.node.ubuntu 18.04')
)
response = client.pool.add(test_ade_pool)
self.assertIsNone(response)
ade_pool = client.pool.get(test_ade_pool.id)
self.assertEqual(ade_pool.virtual_machine_configuration.disk_encryption_configuration.targets,
[models.DiskEncryptionTarget.temporary_disk])
# Test List Pools without Filters
pools = list(client.pool.list())
self.assertTrue(len(pools) > 1)
# Test List Pools with Maximum
options = models.PoolListOptions(max_results=1)
pools = client.pool.list(options)
pools.next()
self.assertEqual(len(pools.current_page), 1)
# Test List Pools with Filter
options = models.PoolListOptions(
filter='startswith(id,\'batch_app_\')',
select='id,state',
expand='stats')
pools = list(client.pool.list(options))
self.assertEqual(len(pools), 1)
pool_id = common.helpers.generate_unique_resource_name(
"{0:}-pool{1:}".format(project_id, job_n))
pool_ids.append(pool_id)
pool_start_commands = ["cd / "]
pool = batchmodels.PoolAddParameter(
id=pool_id,
vm_size=pool_vm_size,
virtual_machine_configuration=batchmodels.
VirtualMachineConfiguration(image_reference=image_ref_to_use,
node_agent_sku_id=sku_to_use),
enable_auto_scale=True,
auto_scale_formula=
"pendingTaskSamplePercent = $PendingTasks.GetSamplePercent(180 * TimeInterval_Second); pendingTaskSamples = pendingTaskSamplePercent < 70 ? 1 : avg($PendingTasks.GetSample(180 * TimeInterval_Second)); $TargetDedicatedNodes = min(100, pendingTaskSamples);",
auto_scale_evaluation_interval=datetime.timedelta(minutes=5),
max_tasks_per_node=1,
task_scheduling_policy=batchmodels.TaskSchedulingPolicy(
node_fill_type="spread"),
start_task=batchmodels.StartTask(
user_identity=batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
elevation_level=batchmodels.ElevationLevel.admin,
scope=batchmodels.AutoUserScope.pool)),
command_line=common.helpers.wrap_commands_in_shell(
"linux", pool_start_commands),
resource_files=[]),
)
common.helpers.create_pool_if_not_exist(batch_client, pool)
# Create job to assign tasks
job_id = "{0:}-job{1:}".format(pool_id, job_n)
job_ids.append(job_id)
job = batchmodels.JobAddParameter(
