def create_pool(self):
from azure.batch import models as batchmodels
pool_id = self.compute_env_prefix + self.machine.name + '-' + str(
self.disk_size)
pool = self.get_pool(pool_id)
if pool is not None:
return pool_id
sku_to_use, image_ref_to_use = self.select_latest_verified_vm_image_with_node_agent_sku(
)
container_configuration = batchmodels.ContainerConfiguration(
container_image_names=[self.image])
config = batchmodels.VirtualMachineConfiguration(
image_reference=image_ref_to_use,
node_agent_sku_id=sku_to_use,
data_disks=[
batchmodels.DataDisk(disk_size_gb=self.disk_size, lun=1)
],
container_configuration=container_configuration,
)
pool = batchmodels.PoolAddParameter(
id=pool_id,
display_name=pool_id,
virtual_machine_configuration=config,
vm_size=self.machine.name,
)
if self.conf[utils.PLATFORM].get('low_priority', False):
pool.target_low_priority_nodes = 1
else:
pool.target_dedicated_nodes = 1
self.batch_client.pool.add(pool)
while self.get_pool(pool_id) is None:
time.sleep(1)
return pool_id
def test_batch_create_pools(self, **kwargs):
client = self.create_sharedkey_client(**kwargs)
# Test List Node Agent SKUs
response = client.account.list_node_agent_skus()
response = list(response)
self.assertTrue(len(response) > 1)
self.assertEqual(response[-1].id, "batch.node.windows amd64")
self.assertEqual(response[-1].os_type.value, "windows")
self.assertTrue(len(response[-1].verified_image_references) > 1)
# Test Create Iaas Pool
users = [
models.UserAccount('test-user-1', 'kt#_gahr!@aGERDXA'),
models.UserAccount('test-user-2', 'kt#_gahr!@aGERDXA', models.ElevationLevel.admin)
]
test_iaas_pool = models.PoolAddParameter(
id=self.get_resource_name('batch_iaas_'),
vm_size='Standard_A1',
virtual_machine_configuration=models.VirtualMachineConfiguration(
image_reference=models.ImageReference(
publisher='MicrosoftWindowsServer',
offer='WindowsServer',
sku='2016-Datacenter-smalldisk'
),
node_agent_sku_id='batch.node.windows amd64',
windows_configuration=models.WindowsConfiguration(True)),
task_scheduling_policy=models.TaskSchedulingPolicy(models.ComputeNodeFillType.pack),
user_accounts=users
)
response = client.pool.add(test_iaas_pool)
self.assertIsNone(response)
# Test list pool node counnt
counts = list(client.account.list_pool_node_counts())
self.assertIsNotNone(counts)
self.assertEqual(len(counts), 1)
self.assertEqual(counts[0].pool_id, test_iaas_pool.id)
self.assertIsNotNone(counts[0].dedicated)
self.assertEqual(counts[0].dedicated.total, 0)
self.assertEqual(counts[0].low_priority.total, 0)
# Test Create Pool with Network Configuration
network_config = models.NetworkConfiguration('/subscriptions/00000000-0000-0000-0000-000000000000'
'/resourceGroups/test'
'/providers/Microsoft.Network'
'/virtualNetworks/vnet1'
'/subnets/subnet1')
test_network_pool = models.PoolAddParameter(
id=self.get_resource_name('batch_network_'),
vm_size='Standard_A1',
network_configuration=network_config,
virtual_machine_configuration=models.VirtualMachineConfiguration(
image_reference=models.ImageReference(
publisher='Canonical',
offer='UbuntuServer',
sku='16.04-LTS'
),
node_agent_sku_id='batch.node.ubuntu 16.04')
)
self.assertBatchError('InvalidPropertyValue', client.pool.add, test_network_pool, models.PoolAddOptions(timeout=45))
# Test Create Pool with Custom Image
test_image_pool = models.PoolAddParameter(
id=self.get_resource_name('batch_image_'),
vm_size='Standard_A1',
virtual_machine_configuration=models.VirtualMachineConfiguration(
image_reference=models.ImageReference(
virtual_machine_image_id="/subscriptions/00000000-0000-0000-0000-000000000000"
"/resourceGroups/test"
"/providers/Microsoft.Compute"
"/images/FakeImage"
),
node_agent_sku_id='batch.node.ubuntu 16.04'
)
)
self.assertBatchError('InvalidPropertyValue', client.pool.add, test_image_pool, models.PoolAddOptions(timeout=45))
# Test Create Pool with OSDisk
os_disk = models.OSDisk(caching=models.CachingType.read_write)
test_osdisk_pool = models.PoolAddParameter(
id=self.get_resource_name('batch_osdisk_'),
vm_size='Standard_A1',
virtual_machine_configuration=models.VirtualMachineConfiguration(
image_reference=models.ImageReference(
publisher='Canonical',
offer='UbuntuServer',
sku='16.04-LTS'
),
node_agent_sku_id='batch.node.ubuntu 16.04',
os_disk=os_disk)
)
response = client.pool.add(test_osdisk_pool)
self.assertIsNone(response)
osdisk_pool = client.pool.get(test_osdisk_pool.id)
self.assertEqual(osdisk_pool.virtual_machine_configuration.os_disk.caching, os_disk.caching)
# Test Create Pool with Data Disk
data_disk = models.DataDisk(lun=1, disk_size_gb=50)
test_disk_pool = models.PoolAddParameter(
id=self.get_resource_name('batch_disk_'),
vm_size='Standard_A1',
virtual_machine_configuration=models.VirtualMachineConfiguration(
image_reference=models.ImageReference(
publisher='Canonical',
offer='UbuntuServer',
sku='16.04-LTS'
),
node_agent_sku_id='batch.node.ubuntu 16.04',
data_disks=[data_disk])
)
response = client.pool.add(test_disk_pool)
self.assertIsNone(response)
disk_pool = client.pool.get(test_disk_pool.id)
self.assertEqual(disk_pool.virtual_machine_configuration.data_disks[0].lun, 1)
self.assertEqual(disk_pool.virtual_machine_configuration.data_disks[0].disk_size_gb, 50)
# Test Create Pool with Application Licenses
test_app_pool = models.PoolAddParameter(
id=self.get_resource_name('batch_app_'),
vm_size='Standard_A1',
application_licenses=["maya"],
virtual_machine_configuration=models.VirtualMachineConfiguration(
image_reference=models.ImageReference(
publisher='Canonical',
offer='UbuntuServer',
sku='16.04-LTS'
),
node_agent_sku_id='batch.node.ubuntu 16.04',
data_disks=[data_disk])
)
response = client.pool.add(test_app_pool)
self.assertIsNone(response)
app_pool = client.pool.get(test_app_pool.id)
self.assertEqual(app_pool.application_licenses[0], "maya")
# Test List Pools without Filters
pools = list(client.pool.list())
self.assertTrue(len(pools) > 1)
# Test List Pools with Maximum
options = models.PoolListOptions(max_results=1)
pools = client.pool.list(options)
pools.next()
self.assertEqual(len(pools.current_page), 1)
# Test List Pools with Filter
options = models.PoolListOptions(
filter='startswith(id,\'batch_app_\')',
select='id,state',
expand='stats')
pools = list(client.pool.list(options))
self.assertEqual(len(pools), 1)
def create_pool(batch_service_client, blob_client, pool_id, config):
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str pool_id: An ID for the new pool.
:param dict config: Configuration details.
"""
if "IMAGE_ID" in os.environ:
image_ref_to_use = batchmodels.ImageReference(
virtual_machine_image_id=os.environ["IMAGE_ID"])
sku_to_use = os.environ["SKU"]
else:
sku_to_use, image_ref_to_use = \
select_latest_verified_vm_image_with_node_agent_sku(
batch_service_client,
config['node_os_publisher'],
config['node_os_offer'],
config['node_os_sku'])
account_name = os.environ['AZURE_STORAGE_ACCOUNT']
account_key = _get_blob_key(account_name)
# Get the node agent SKU and image reference for the virtual machine
# configuration.
# For more information about the virtual machine configuration, see:
# https://azure.microsoft.com/documentation/articles/batch-linux-nodes/
start_vm_commands = None
if config.get('create_vm_commands', None):
start_vm_commands = _create_commands(config['create_vm_commands'])
start_vm_commands = [
command.format(accountname=account_name, accountkey=account_key)
for command in start_vm_commands
]
data_disks = None
if config['data_disk_sizes']:
data_disks = []
for i, disk_size in config['data_disk_sizes'].iteritems():
data_disks.append(batchmodels.DataDisk(i, disk_size))
resource_files = []
if config['start_resources']:
for vm_file_name, blob_path in config['start_resources'].iteritems():
container_name, blob_name = unpack_path(blob_path)
res_url = generate_blob_url(blob_client, container_name, blob_name)
resource_files.append(
batch.models.ResourceFile(res_url, vm_file_name))
user = batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)
new_pool = batch.models.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=image_ref_to_use,
node_agent_sku_id=sku_to_use,
data_disks=data_disks),
vm_size=config['pool_vm_size'],
enable_auto_scale=True,
auto_scale_formula=config['auto_scale_formula'],
auto_scale_evaluation_interval=datetime.timedelta(minutes=5),
start_task=batch.models.StartTask(
command_line=wrap_commands_in_shell('linux', start_vm_commands),
user_identity=batchmodels.UserIdentity(auto_user=user),
resource_files=resource_files,
wait_for_success=True),
max_tasks_per_node=config['max_tasks_per_node'],
)
try:
batch_service_client.pool.add(new_pool)
except batchmodels.batch_error.BatchErrorException as err:
print_batch_exception(err)
raise
def test_batch_create_pools(self, **kwargs):
client = self.create_sharedkey_client(**kwargs)
# Test List Node Agent SKUs
response = client.account.list_supported_images()
response = list(response)
self.assertTrue(len(response) > 1)
self.assertIsNotNone(response[-1].image_reference)
# Test Create Iaas Pool
users = [
models.UserAccount(name='test-user-1', password='******'),
models.UserAccount(name='test-user-2', password='******', elevation_level=models.ElevationLevel.admin)
]
test_iaas_pool = models.PoolAddParameter(
id=self.get_resource_name('batch_iaas_'),
vm_size=DEFAULT_VM_SIZE,
virtual_machine_configuration=models.VirtualMachineConfiguration(
image_reference=models.ImageReference(
publisher='MicrosoftWindowsServer',
offer='WindowsServer',
sku='2016-Datacenter-smalldisk'
),
node_agent_sku_id='batch.node.windows amd64',
windows_configuration=models.WindowsConfiguration(enable_automatic_updates=True)),
task_scheduling_policy=models.TaskSchedulingPolicy(node_fill_type=models.ComputeNodeFillType.pack),
user_accounts=users
)
response = client.pool.add(test_iaas_pool)
self.assertIsNone(response)
# Test list pool node counnt
counts = list(client.account.list_pool_node_counts())
self.assertIsNotNone(counts)
self.assertEqual(len(counts), 1)
self.assertEqual(counts[0].pool_id, test_iaas_pool.id)
self.assertIsNotNone(counts[0].dedicated)
self.assertEqual(counts[0].dedicated.total, 0)
self.assertEqual(counts[0].dedicated.leaving_pool, 0)
self.assertEqual(counts[0].low_priority.total, 0)
# Test Create Pool with Network Configuration
#TODO Public IP tests
network_config = models.NetworkConfiguration(subnet_id='/subscriptions/00000000-0000-0000-0000-000000000000'
'/resourceGroups/test'
'/providers/Microsoft.Network'
'/virtualNetworks/vnet1'
'/subnets/subnet1')
test_network_pool = models.PoolAddParameter(
id=self.get_resource_name('batch_network_'),
vm_size=DEFAULT_VM_SIZE,
network_configuration=network_config,
virtual_machine_configuration=models.VirtualMachineConfiguration(
image_reference=models.ImageReference(
publisher='Canonical',
offer='UbuntuServer',
sku='18.04-LTS'
),
node_agent_sku_id='batch.node.ubuntu 18.04')
)
self.assertBatchError('InvalidPropertyValue', client.pool.add, test_network_pool, models.PoolAddOptions(timeout=45))
test_image_pool = models.PoolAddParameter(
id=self.get_resource_name('batch_image_'),
vm_size=DEFAULT_VM_SIZE,
virtual_machine_configuration=models.VirtualMachineConfiguration(
image_reference=models.ImageReference(
virtual_machine_image_id="/subscriptions/00000000-0000-0000-0000-000000000000"
"/resourceGroups/test"
"/providers/Microsoft.Compute"
"/gallery/FakeGallery"
"/images/FakeImage"
"/versions/version"
),
node_agent_sku_id='batch.node.ubuntu 18.04'
)
)
self.assertBatchError('InvalidPropertyValue', client.pool.add, test_image_pool, models.PoolAddOptions(timeout=45))
# Test Create Pool with Data Disk
data_disk = models.DataDisk(lun=1, disk_size_gb=50)
test_disk_pool = models.PoolAddParameter(
id=self.get_resource_name('batch_disk_'),
vm_size=DEFAULT_VM_SIZE,
virtual_machine_configuration=models.VirtualMachineConfiguration(
image_reference=models.ImageReference(
publisher='Canonical',
offer='UbuntuServer',
sku='18.04-LTS'
),
node_agent_sku_id='batch.node.ubuntu 18.04',
data_disks=[data_disk])
)
response = client.pool.add(test_disk_pool)
self.assertIsNone(response)
disk_pool = client.pool.get(test_disk_pool.id)
self.assertEqual(disk_pool.virtual_machine_configuration.data_disks[0].lun, 1)
self.assertEqual(disk_pool.virtual_machine_configuration.data_disks[0].disk_size_gb, 50)
# Test Create Pool with Application Licenses
test_app_pool = models.PoolAddParameter(
id=self.get_resource_name('batch_app_'),
vm_size=DEFAULT_VM_SIZE,
application_licenses=["maya"],
virtual_machine_configuration=models.VirtualMachineConfiguration(
image_reference=models.ImageReference(
publisher='Canonical',
offer='UbuntuServer',
sku='18.04-LTS'
),
node_agent_sku_id='batch.node.ubuntu 18.04',
data_disks=[data_disk])
)
response = client.pool.add(test_app_pool)
self.assertIsNone(response)
app_pool = client.pool.get(test_app_pool.id)
self.assertEqual(app_pool.application_licenses[0], "maya")
# Test Create Pool with Azure Disk Encryption
test_ade_pool = models.PoolAddParameter(
id=self.get_resource_name('batch_ade_'),
vm_size=DEFAULT_VM_SIZE,
virtual_machine_configuration=models.VirtualMachineConfiguration(
image_reference=models.ImageReference(
publisher='Canonical',
offer='UbuntuServer',
sku='18.04-LTS'
),
disk_encryption_configuration=models.DiskEncryptionConfiguration(
targets=[models.DiskEncryptionTarget.temporary_disk]
),
node_agent_sku_id='batch.node.ubuntu 18.04')
)
response = client.pool.add(test_ade_pool)
self.assertIsNone(response)
ade_pool = client.pool.get(test_ade_pool.id)
self.assertEqual(ade_pool.virtual_machine_configuration.disk_encryption_configuration.targets,
[models.DiskEncryptionTarget.temporary_disk])
# Test List Pools without Filters
pools = list(client.pool.list())
self.assertTrue(len(pools) > 1)
# Test List Pools with Maximum
options = models.PoolListOptions(max_results=1)
pools = client.pool.list(options)
pools.next()
self.assertEqual(len(pools.current_page), 1)
# Test List Pools with Filter
options = models.PoolListOptions(
filter='startswith(id,\'batch_app_\')',
select='id,state',
expand='stats')
pools = list(client.pool.list(options))
self.assertEqual(len(pools), 1)