Пример #1
0
    def create_pool(self):
        from azure.batch import models as batchmodels

        pool_id = self.compute_env_prefix + self.machine.name + '-' + str(
            self.disk_size)

        pool = self.get_pool(pool_id)
        if pool is not None:
            return pool_id

        sku_to_use, image_ref_to_use = self.select_latest_verified_vm_image_with_node_agent_sku(
        )

        container_configuration = batchmodels.ContainerConfiguration(
            container_image_names=[self.image])

        config = batchmodels.VirtualMachineConfiguration(
            image_reference=image_ref_to_use,
            node_agent_sku_id=sku_to_use,
            data_disks=[
                batchmodels.DataDisk(disk_size_gb=self.disk_size, lun=1)
            ],
            container_configuration=container_configuration,
        )

        pool = batchmodels.PoolAddParameter(
            id=pool_id,
            display_name=pool_id,
            virtual_machine_configuration=config,
            vm_size=self.machine.name,
        )

        if self.conf[utils.PLATFORM].get('low_priority', False):
            pool.target_low_priority_nodes = 1
        else:
            pool.target_dedicated_nodes = 1

        self.batch_client.pool.add(pool)

        while self.get_pool(pool_id) is None:
            time.sleep(1)

        return pool_id
    def test_batch_create_pools(self, **kwargs):
        client = self.create_sharedkey_client(**kwargs)
        # Test List Node Agent SKUs
        response = client.account.list_node_agent_skus()
        response = list(response)
        self.assertTrue(len(response) > 1)
        self.assertEqual(response[-1].id, "batch.node.windows amd64")
        self.assertEqual(response[-1].os_type.value, "windows")
        self.assertTrue(len(response[-1].verified_image_references) > 1)

        # Test Create Iaas Pool
        users = [
            models.UserAccount('test-user-1', 'kt#_gahr!@aGERDXA'),
            models.UserAccount('test-user-2', 'kt#_gahr!@aGERDXA', models.ElevationLevel.admin)
        ]
        test_iaas_pool = models.PoolAddParameter(
            id=self.get_resource_name('batch_iaas_'),
            vm_size='Standard_A1',
            virtual_machine_configuration=models.VirtualMachineConfiguration(
                image_reference=models.ImageReference(
                    publisher='MicrosoftWindowsServer',
                    offer='WindowsServer',
                    sku='2016-Datacenter-smalldisk'
                ),
                node_agent_sku_id='batch.node.windows amd64',
                windows_configuration=models.WindowsConfiguration(True)),
            task_scheduling_policy=models.TaskSchedulingPolicy(models.ComputeNodeFillType.pack),
            user_accounts=users
        )
        response = client.pool.add(test_iaas_pool)
        self.assertIsNone(response)

        # Test list pool node counnt
        counts = list(client.account.list_pool_node_counts())
        self.assertIsNotNone(counts)
        self.assertEqual(len(counts), 1)
        self.assertEqual(counts[0].pool_id, test_iaas_pool.id)
        self.assertIsNotNone(counts[0].dedicated)
        self.assertEqual(counts[0].dedicated.total, 0)
        self.assertEqual(counts[0].low_priority.total, 0)

        # Test Create Pool with Network Configuration
        network_config = models.NetworkConfiguration('/subscriptions/00000000-0000-0000-0000-000000000000'
                                                     '/resourceGroups/test'
                                                     '/providers/Microsoft.Network'
                                                     '/virtualNetworks/vnet1'
                                                     '/subnets/subnet1')
        test_network_pool = models.PoolAddParameter(
            id=self.get_resource_name('batch_network_'),
            vm_size='Standard_A1',
            network_configuration=network_config,
            virtual_machine_configuration=models.VirtualMachineConfiguration(
                image_reference=models.ImageReference(
                    publisher='Canonical',
                    offer='UbuntuServer',
                    sku='16.04-LTS'
                ),
                node_agent_sku_id='batch.node.ubuntu 16.04')
        )
        self.assertBatchError('InvalidPropertyValue', client.pool.add, test_network_pool, models.PoolAddOptions(timeout=45))

        # Test Create Pool with Custom Image
        test_image_pool = models.PoolAddParameter(
            id=self.get_resource_name('batch_image_'),
            vm_size='Standard_A1',
            virtual_machine_configuration=models.VirtualMachineConfiguration(
                image_reference=models.ImageReference(
                    virtual_machine_image_id="/subscriptions/00000000-0000-0000-0000-000000000000"
                                             "/resourceGroups/test"
                                             "/providers/Microsoft.Compute"
                                             "/images/FakeImage"
                ),
                node_agent_sku_id='batch.node.ubuntu 16.04'
            )
        )
        self.assertBatchError('InvalidPropertyValue', client.pool.add, test_image_pool, models.PoolAddOptions(timeout=45))

        # Test Create Pool with OSDisk
        os_disk = models.OSDisk(caching=models.CachingType.read_write)
        test_osdisk_pool = models.PoolAddParameter(
            id=self.get_resource_name('batch_osdisk_'),
            vm_size='Standard_A1',
            virtual_machine_configuration=models.VirtualMachineConfiguration(
                image_reference=models.ImageReference(
                    publisher='Canonical',
                    offer='UbuntuServer',
                    sku='16.04-LTS'
                ),
                node_agent_sku_id='batch.node.ubuntu 16.04',
                os_disk=os_disk)
        )
        response = client.pool.add(test_osdisk_pool)
        self.assertIsNone(response)
        osdisk_pool = client.pool.get(test_osdisk_pool.id)
        self.assertEqual(osdisk_pool.virtual_machine_configuration.os_disk.caching, os_disk.caching)

        # Test Create Pool with Data Disk
        data_disk = models.DataDisk(lun=1, disk_size_gb=50)
        test_disk_pool = models.PoolAddParameter(
            id=self.get_resource_name('batch_disk_'),
            vm_size='Standard_A1',
            virtual_machine_configuration=models.VirtualMachineConfiguration(
                image_reference=models.ImageReference(
                    publisher='Canonical',
                    offer='UbuntuServer',
                    sku='16.04-LTS'
                ),
                node_agent_sku_id='batch.node.ubuntu 16.04',
                data_disks=[data_disk])
        )
        response = client.pool.add(test_disk_pool)
        self.assertIsNone(response)
        disk_pool = client.pool.get(test_disk_pool.id)
        self.assertEqual(disk_pool.virtual_machine_configuration.data_disks[0].lun, 1)
        self.assertEqual(disk_pool.virtual_machine_configuration.data_disks[0].disk_size_gb, 50)

        # Test Create Pool with Application Licenses
        test_app_pool = models.PoolAddParameter(
            id=self.get_resource_name('batch_app_'),
            vm_size='Standard_A1',
            application_licenses=["maya"],
            virtual_machine_configuration=models.VirtualMachineConfiguration(
                image_reference=models.ImageReference(
                    publisher='Canonical',
                    offer='UbuntuServer',
                    sku='16.04-LTS'
                ),
                node_agent_sku_id='batch.node.ubuntu 16.04',
                data_disks=[data_disk])
        )
        response = client.pool.add(test_app_pool)
        self.assertIsNone(response)
        app_pool = client.pool.get(test_app_pool.id)
        self.assertEqual(app_pool.application_licenses[0], "maya")

        # Test List Pools without Filters
        pools = list(client.pool.list())
        self.assertTrue(len(pools) > 1)

        # Test List Pools with Maximum
        options = models.PoolListOptions(max_results=1)
        pools = client.pool.list(options)
        pools.next()
        self.assertEqual(len(pools.current_page), 1)

        # Test List Pools with Filter
        options = models.PoolListOptions(
            filter='startswith(id,\'batch_app_\')',
            select='id,state',
            expand='stats')
        pools = list(client.pool.list(options))
        self.assertEqual(len(pools), 1)
Пример #3
0
def create_pool(batch_service_client, blob_client, pool_id, config):
    """
    Creates a pool of compute nodes with the specified OS settings.

    :param batch_service_client: A Batch service client.
    :type batch_service_client: `azure.batch.BatchServiceClient`
    :param str pool_id: An ID for the new pool.
    :param dict config: Configuration details.
    """

    if "IMAGE_ID" in os.environ:
        image_ref_to_use = batchmodels.ImageReference(
            virtual_machine_image_id=os.environ["IMAGE_ID"])
        sku_to_use = os.environ["SKU"]
    else:
        sku_to_use, image_ref_to_use = \
            select_latest_verified_vm_image_with_node_agent_sku(
                batch_service_client,
                config['node_os_publisher'],
                config['node_os_offer'],
                config['node_os_sku'])
    account_name = os.environ['AZURE_STORAGE_ACCOUNT']
    account_key = _get_blob_key(account_name)

    # Get the node agent SKU and image reference for the virtual machine
    # configuration.
    # For more information about the virtual machine configuration, see:
    # https://azure.microsoft.com/documentation/articles/batch-linux-nodes/

    start_vm_commands = None
    if config.get('create_vm_commands', None):
        start_vm_commands = _create_commands(config['create_vm_commands'])
        start_vm_commands = [
            command.format(accountname=account_name, accountkey=account_key)
            for command in start_vm_commands
        ]

    data_disks = None
    if config['data_disk_sizes']:
        data_disks = []
        for i, disk_size in config['data_disk_sizes'].iteritems():
            data_disks.append(batchmodels.DataDisk(i, disk_size))

    resource_files = []

    if config['start_resources']:
        for vm_file_name, blob_path in config['start_resources'].iteritems():
            container_name, blob_name = unpack_path(blob_path)
            res_url = generate_blob_url(blob_client, container_name, blob_name)
            resource_files.append(
                batch.models.ResourceFile(res_url, vm_file_name))

    user = batchmodels.AutoUserSpecification(
        scope=batchmodels.AutoUserScope.pool,
        elevation_level=batchmodels.ElevationLevel.admin)
    new_pool = batch.models.PoolAddParameter(
        id=pool_id,
        virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
            image_reference=image_ref_to_use,
            node_agent_sku_id=sku_to_use,
            data_disks=data_disks),
        vm_size=config['pool_vm_size'],
        enable_auto_scale=True,
        auto_scale_formula=config['auto_scale_formula'],
        auto_scale_evaluation_interval=datetime.timedelta(minutes=5),
        start_task=batch.models.StartTask(
            command_line=wrap_commands_in_shell('linux', start_vm_commands),
            user_identity=batchmodels.UserIdentity(auto_user=user),
            resource_files=resource_files,
            wait_for_success=True),
        max_tasks_per_node=config['max_tasks_per_node'],
    )

    try:
        batch_service_client.pool.add(new_pool)
    except batchmodels.batch_error.BatchErrorException as err:
        print_batch_exception(err)
        raise
    def test_batch_create_pools(self, **kwargs):
        client = self.create_sharedkey_client(**kwargs)
        # Test List Node Agent SKUs
        response = client.account.list_supported_images()
        response = list(response)
        self.assertTrue(len(response) > 1)
        self.assertIsNotNone(response[-1].image_reference)

        # Test Create Iaas Pool
        users = [
            models.UserAccount(name='test-user-1', password='******'),
            models.UserAccount(name='test-user-2', password='******', elevation_level=models.ElevationLevel.admin)
        ]
        test_iaas_pool = models.PoolAddParameter(
            id=self.get_resource_name('batch_iaas_'),
            vm_size=DEFAULT_VM_SIZE,
            virtual_machine_configuration=models.VirtualMachineConfiguration(
                image_reference=models.ImageReference(
                    publisher='MicrosoftWindowsServer',
                    offer='WindowsServer',
                    sku='2016-Datacenter-smalldisk'
                ),
                node_agent_sku_id='batch.node.windows amd64',
                windows_configuration=models.WindowsConfiguration(enable_automatic_updates=True)),
            task_scheduling_policy=models.TaskSchedulingPolicy(node_fill_type=models.ComputeNodeFillType.pack),
            user_accounts=users
        )
        response = client.pool.add(test_iaas_pool)
        self.assertIsNone(response)

        # Test list pool node counnt
        counts = list(client.account.list_pool_node_counts())
        self.assertIsNotNone(counts)
        self.assertEqual(len(counts), 1)
        self.assertEqual(counts[0].pool_id, test_iaas_pool.id)
        self.assertIsNotNone(counts[0].dedicated)
        self.assertEqual(counts[0].dedicated.total, 0)
        self.assertEqual(counts[0].dedicated.leaving_pool, 0)
        self.assertEqual(counts[0].low_priority.total, 0)

        # Test Create Pool with Network Configuration
        #TODO Public IP tests
        network_config = models.NetworkConfiguration(subnet_id='/subscriptions/00000000-0000-0000-0000-000000000000'
                                                     '/resourceGroups/test'
                                                     '/providers/Microsoft.Network'
                                                     '/virtualNetworks/vnet1'
                                                     '/subnets/subnet1')
        test_network_pool = models.PoolAddParameter(
            id=self.get_resource_name('batch_network_'),
            vm_size=DEFAULT_VM_SIZE,
            network_configuration=network_config,
            virtual_machine_configuration=models.VirtualMachineConfiguration(
                image_reference=models.ImageReference(
                    publisher='Canonical',
                    offer='UbuntuServer',
                    sku='18.04-LTS'
                ),
                node_agent_sku_id='batch.node.ubuntu 18.04')
        )
        self.assertBatchError('InvalidPropertyValue', client.pool.add, test_network_pool, models.PoolAddOptions(timeout=45))

        test_image_pool = models.PoolAddParameter(
            id=self.get_resource_name('batch_image_'),
            vm_size=DEFAULT_VM_SIZE,
            virtual_machine_configuration=models.VirtualMachineConfiguration(
                image_reference=models.ImageReference(
                    virtual_machine_image_id="/subscriptions/00000000-0000-0000-0000-000000000000"
                                             "/resourceGroups/test"
                                             "/providers/Microsoft.Compute"
                                             "/gallery/FakeGallery"
                                             "/images/FakeImage"
                                             "/versions/version"
                ),
                node_agent_sku_id='batch.node.ubuntu 18.04'
            )
        )
        self.assertBatchError('InvalidPropertyValue', client.pool.add, test_image_pool, models.PoolAddOptions(timeout=45))

        # Test Create Pool with Data Disk
        data_disk = models.DataDisk(lun=1, disk_size_gb=50)
        test_disk_pool = models.PoolAddParameter(
            id=self.get_resource_name('batch_disk_'),
            vm_size=DEFAULT_VM_SIZE,
            virtual_machine_configuration=models.VirtualMachineConfiguration(
                image_reference=models.ImageReference(
                    publisher='Canonical',
                    offer='UbuntuServer',
                    sku='18.04-LTS'
                ),
                node_agent_sku_id='batch.node.ubuntu 18.04',
                data_disks=[data_disk])
        )
        response = client.pool.add(test_disk_pool)
        self.assertIsNone(response)
        disk_pool = client.pool.get(test_disk_pool.id)
        self.assertEqual(disk_pool.virtual_machine_configuration.data_disks[0].lun, 1)
        self.assertEqual(disk_pool.virtual_machine_configuration.data_disks[0].disk_size_gb, 50)

        # Test Create Pool with Application Licenses
        test_app_pool = models.PoolAddParameter(
            id=self.get_resource_name('batch_app_'),
            vm_size=DEFAULT_VM_SIZE,
            application_licenses=["maya"],
            virtual_machine_configuration=models.VirtualMachineConfiguration(
                image_reference=models.ImageReference(
                    publisher='Canonical',
                    offer='UbuntuServer',
                    sku='18.04-LTS'
                ),
                node_agent_sku_id='batch.node.ubuntu 18.04',
                data_disks=[data_disk])
        )
        response = client.pool.add(test_app_pool)
        self.assertIsNone(response)
        app_pool = client.pool.get(test_app_pool.id)
        self.assertEqual(app_pool.application_licenses[0], "maya")

        # Test Create Pool with Azure Disk Encryption
        test_ade_pool = models.PoolAddParameter(
            id=self.get_resource_name('batch_ade_'),
            vm_size=DEFAULT_VM_SIZE,
            virtual_machine_configuration=models.VirtualMachineConfiguration(
                image_reference=models.ImageReference(
                    publisher='Canonical',
                    offer='UbuntuServer',
                    sku='18.04-LTS'
                ),
                disk_encryption_configuration=models.DiskEncryptionConfiguration(
                    targets=[models.DiskEncryptionTarget.temporary_disk]
                ),
                node_agent_sku_id='batch.node.ubuntu 18.04')
        )
        response = client.pool.add(test_ade_pool)
        self.assertIsNone(response)
        ade_pool = client.pool.get(test_ade_pool.id)
        self.assertEqual(ade_pool.virtual_machine_configuration.disk_encryption_configuration.targets,
                         [models.DiskEncryptionTarget.temporary_disk])

        # Test List Pools without Filters
        pools = list(client.pool.list())
        self.assertTrue(len(pools) > 1)

        # Test List Pools with Maximum
        options = models.PoolListOptions(max_results=1)
        pools = client.pool.list(options)
        pools.next()
        self.assertEqual(len(pools.current_page), 1)

        # Test List Pools with Filter
        options = models.PoolListOptions(
            filter='startswith(id,\'batch_app_\')',
            select='id,state',
            expand='stats')
        pools = list(client.pool.list(options))
        self.assertEqual(len(pools), 1)