def pool_create(): image_reference = batchmodel.ImageReference( publisher=config_azure['batch_pool_image_publisher'], offer=config_azure['batch_pool_image_offer'], sku=config_azure['batch_pool_image_sku']) vm_config = batchmodel.VirtualMachineConfiguration( image_reference=image_reference, node_agent_sku_id=config_azure['batch_pool_node_agent_sku']) vm_start_task = batchmodel.StartTask( command_line= '/bin/bash -c "sudo yum -y install epel-release; sudo yum -y install python36 python36-devel python36-tools; sudo python36 -m ensurepip; sudo yum -y install openmpi openmpi-devel; sudo env MPICC=/usr/lib64/openmpi/bin/mpicc pip3 install mpi4py numpy; sudo pip3 --yes uninstall azure azure-common azure-storage; sudo pip3 install azure-storage azure-batch"', user_identity=batchmodel.UserIdentity( auto_user=batchmodel.AutoUserSpecification( scope=batchmodel.AutoUserScope.pool, elevation_level=batchmodel.ElevationLevel.admin)), wait_for_success=True) batch_service.pool.add(pool=batchmodel.PoolAddParameter( id=config_azure['batch_pool_name'], vm_size=config_azure['batch_pool_vm_size'], virtual_machine_configuration=vm_config, target_dedicated_nodes=config_azure[ 'batch_pool_target_dedicated_nodes'], enable_inter_node_communication=True, start_task=vm_start_task), raw=True)
def create_pool(batch_client, name_pool, number_nodes=0, cmd_s_task=None, rule_scale_pool=None): #parameter image node param_image = models.VirtualMachineConfiguration( image_reference=models.ImageReference(offer='UbuntuServer', publisher='Canonical', sku='18.04-LTS', version='latest', virtual_machine_image_id=None), node_agent_sku_id='batch.node.ubuntu 18.04') #parameter pool new_pool = models.PoolAddParameter( id=name_pool, vm_size='standard_d1_v2', #target_dedicated_nodes = number_nodes, virtual_machine_configuration=param_image, enable_inter_node_communication=True, enable_auto_scale=True, auto_scale_formula=rule_scale_pool, auto_scale_evaluation_interval='PT5M' #start_task = s_task ) batch_client.pool.add(new_pool)
def test_batch_create_pool_with_blobfuse_mount(self, **kwargs): client = self.create_sharedkey_client(**kwargs) # Test Create Iaas Pool test_iaas_pool = models.PoolAddParameter( id=self.get_resource_name('batch_iaas_'), vm_size='Standard_A1', virtual_machine_configuration=models.VirtualMachineConfiguration( image_reference=models.ImageReference( publisher='MicrosoftWindowsServer', offer='WindowsServer', sku='2016-Datacenter-smalldisk' ), node_agent_sku_id='batch.node.windows amd64', windows_configuration=models.WindowsConfiguration(enable_automatic_updates=True)), task_scheduling_policy=models.TaskSchedulingPolicy(node_fill_type=models.ComputeNodeFillType.pack), mount_configuration=[models.MountConfiguration( azure_blob_file_system_configuration=models.AzureBlobFileSystemConfiguration( account_name='test', container_name='https://test.blob.core.windows.net:443/test-container', relative_mount_path='foo', account_key='fake_account_key' ) )] ) response = client.pool.add(test_iaas_pool) self.assertIsNone(response) mount_pool = client.pool.get(test_iaas_pool.id) self.assertIsNotNone(mount_pool.mount_configuration) self.assertEqual(len(mount_pool.mount_configuration), 1) self.assertIsNotNone(mount_pool.mount_configuration[0].azure_blob_file_system_configuration) self.assertIsNone(mount_pool.mount_configuration[0].nfs_mount_configuration)
def __create_pool_and_job(self, cluster_conf: models.ClusterConfiguration, software_metadata_key: str, start_task, VmImageModel): """ Create a pool and job :param cluster_conf: the configuration object used to create the cluster :type cluster_conf: aztk.models.ClusterConfiguration :parm software_metadata_key: the id of the software being used on the cluster :param start_task: the start task for the cluster :param VmImageModel: the type of image to provision for the cluster :param wait: wait until the cluster is ready """ self._get_cluster_data(cluster_conf.cluster_id).save_cluster_config(cluster_conf) # reuse pool_id as job_id pool_id = cluster_conf.cluster_id job_id = cluster_conf.cluster_id # Get a verified node agent sku sku_to_use, image_ref_to_use = \ helpers.select_latest_verified_vm_image_with_node_agent_sku( VmImageModel.publisher, VmImageModel.offer, VmImageModel.sku, self.batch_client) network_conf = None if cluster_conf.subnet_id is not None: network_conf = batch_models.NetworkConfiguration( subnet_id=cluster_conf.subnet_id) auto_scale_formula = "$TargetDedicatedNodes={0}; $TargetLowPriorityNodes={1}".format( cluster_conf.vm_count, cluster_conf.vm_low_pri_count) # Confiure the pool pool = batch_models.PoolAddParameter( id=pool_id, virtual_machine_configuration=batch_models.VirtualMachineConfiguration( image_reference=image_ref_to_use, node_agent_sku_id=sku_to_use), vm_size=cluster_conf.vm_size, enable_auto_scale=True, auto_scale_formula=auto_scale_formula, auto_scale_evaluation_interval=timedelta(minutes=5), start_task=start_task, enable_inter_node_communication=True if not cluster_conf.subnet_id else False, max_tasks_per_node=4, network_configuration=network_conf, metadata=[ batch_models.MetadataItem( name=constants.AZTK_SOFTWARE_METADATA_KEY, value=software_metadata_key), batch_models.MetadataItem( name=constants.AZTK_MODE_METADATA_KEY, value=constants.AZTK_CLUSTER_MODE_METADATA) ]) # Create the pool + create user for the pool helpers.create_pool_if_not_exist(pool, self.batch_client) # Create job job = batch_models.JobAddParameter( id=job_id, pool_info=batch_models.PoolInformation(pool_id=pool_id)) # Add job to batch self.batch_client.job.add(job) return helpers.get_cluster(cluster_conf.cluster_id, self.batch_client)
def create_pool(self, size, name): """Create and deploy a new pool. Called on job submission by submission.py. TODO: Support auto-scale formula. """ image = self.environment.get_image() node_agent_sku_id = image.pop('node_sku_id') pool_id = 'Maya_Pool_{}'.format(uuid.uuid4()) pool_config = models.VirtualMachineConfiguration( image_reference=models.ImageReference(**image), node_agent_sku_id=node_agent_sku_id) self._log.info("Creating new pool '{}' with {} VMs.".format( name, size)) new_pool = models.PoolAddParameter( id=pool_id, display_name="Maya Pool for {}".format(name), resize_timeout=datetime.timedelta(minutes=30), application_licenses=self.environment.get_application_licenses(), vm_size=self.environment.get_vm_sku(), virtual_machine_configuration=pool_config, target_dedicated_nodes=int(size[0]), target_low_priority_nodes=int(size[1]), max_tasks_per_node=1) self._call(self.batch.pool.add, new_pool) self._log.debug("Successfully created pool.") return {"poolId": pool_id}
def create_pool(batch_client, pool_id, vm_size, vm_count, app_files): """Creates an Azure Batch pool with the specified id. :param batch_client: The batch client to use. :type batch_client: `batchserviceclient.BatchServiceClient` :param block_blob_client: The storage block blob client to use. :type block_blob_client: `azure.storage.blob.BlockBlobService` :param str pool_id: The id of the pool to create. :param str vm_size: vm size (sku) :param int vm_count: number of vms to allocate :param list app_files: The list of all the other scripts to upload. """ # pick the latest supported 16.04 sku for UbuntuServer sku_to_use, image_ref_to_use = \ common.helpers.select_latest_verified_vm_image_with_node_agent_sku( batch_client, 'Canonical', 'UbuntuServer', '14.04') user = batchmodels.AutoUserSpecification( scope=batchmodels.AutoUserScope.pool, elevation_level=batchmodels.ElevationLevel.admin) task_commands = get_list_from_file('configs/start_commands') print(task_commands) pool = batchmodels.PoolAddParameter( id=pool_id, virtual_machine_configuration=batchmodels.VirtualMachineConfiguration( image_reference=image_ref_to_use, node_agent_sku_id=sku_to_use), vm_size=vm_size, target_dedicated=vm_count, start_task=batchmodels.StartTask( command_line=common.helpers.wrap_commands_in_shell( 'linux', task_commands), user_identity=batchmodels.UserIdentity(auto_user=user), resource_files=app_files, wait_for_success=True)) common.helpers.create_pool_if_not_exist(batch_client, pool)
def create_pool(batch_service_client: batch.BatchServiceClient, config: Dict[str, str]) -> None: """ Creates a pool of compute nodes with the specified OS settings. :param batch_service_client: A Batch service client. :type batch_service_client: `azure.batch.BatchServiceClient` :param config: Configuration """ print("Creating pool [{}]...".format(config['POOL_ID'])) # Create a new pool of Linux compute nodes using an Azure Virtual Machines # Marketplace image. For more information about creating pools of Linux # nodes, see: # https://azure.microsoft.com/documentation/articles/batch-linux-nodes/ new_pool = batch.models.PoolAddParameter( id=config['POOL_ID'], virtual_machine_configuration=batchmodels.VirtualMachineConfiguration( image_reference=batchmodels.ImageReference( publisher="microsoft-azure-batch", offer="ubuntu-server-container", sku='16-04-lts', version="latest", ), node_agent_sku_id="batch.node.ubuntu 16.04", container_configuration=create_container_config(config), ), vm_size=config['POOL_VM_SIZE'], target_dedicated_nodes=config['POOL_NODE_COUNT'], task_slots_per_node=config['TASK_SLOTS_PER_NODE'], ) batch_service_client.pool.add(new_pool)
def create_pool(batch_service_client, pool_id, vm_size, imageName, versions, auto_scale_formula): """ Creates a pool of compute nodes with the specified OS settings. :param batch_service_client: A Batch service client. :type batch_service_client: `azure.batch.BatchServiceClient` :param str pool_id: An ID for the new pool. :param str publisher: Marketplace image publisher :param str offer: Marketplace image offer :param str sku: Marketplace image sky """ print('Creating pool [{}]...'.format(pool_id)) new_pool = batch.models.PoolAddParameter( id=pool_id, virtual_machine_configuration=batchmodels.VirtualMachineConfiguration( image_reference=batchmodels.ImageReference( virtual_machine_image_id="/subscriptions/{}/resourceGroups/{}/providers/Microsoft.Compute/images/{}".format( "ad49354a-6ce2-4dae-a51d-b6907372f608", "BrowseCloud", imageName) ), node_agent_sku_id="batch.node.windows amd64"), vm_size=vm_size, start_task=None, enable_auto_scale=True, auto_scale_formula=auto_scale_formula, application_package_references=[batchmodels.ApplicationPackageReference( application_id="browsecloudtrainer", version=version) for version in versions], auto_scale_evaluation_interval=timedelta( minutes=5) # the smallest evaluation interval ) batch_service_client.pool.add(new_pool)
def create_pool(batch_client, block_blob_client, pool_id, vm_size, vm_count): """Creates an Azure Batch pool with the specified id. :param batch_client: The batch client to use. :type batch_client: `batchserviceclient.BatchServiceClient` :param block_blob_client: The storage block blob client to use. :type block_blob_client: `azure.storage.blob.BlockBlobService` :param str pool_id: The id of the pool to create. :param str vm_size: vm size (sku) :param int vm_count: number of vms to allocate """ #application_packages = [ # batchmodels.ApplicationPackageReference(application_id='Regridv3', version='1.0.0'), # batchmodels.ApplicationPackageReference(application_id='us_5km', version='1.0.0'), # batchmodels.ApplicationPackageReference(application_id='us_800m_t6', version='1.0.0'), #] sku_to_use, image_ref_to_use = \ common.helpers.select_latest_verified_vm_image_with_node_agent_sku( batch_client, 'Canonical', 'UbuntuServer', '18.04') #image_ref_to_use = batchmodels.ImageReference( # publisher = 'microsoftwindowsserver', offer = 'windowsserver', sku = '2019-datacenter', version = 'latest' #) #sku_to_use = "batch.node.windows amd64" pool = batchmodels.PoolAddParameter( id=pool_id, virtual_machine_configuration=batchmodels.VirtualMachineConfiguration( image_reference=image_ref_to_use, node_agent_sku_id=sku_to_use), vm_size=vm_size, max_tasks_per_node=_POOL_MAX_TASKS_PER_NODE, #application_package_references=application_packages, target_dedicated_nodes=vm_count) common.helpers.create_pool_if_not_exist(batch_client, pool)
def create_pool(batch_service_client, pool_id): """ Creates a pool of compute nodes with the specified OS settings. :param batch_service_client: A Batch service client. :type batch_service_client: `azure.batch.BatchServiceClient` :param str pool_id: An ID for the new pool. :param str publisher: Marketplace image publisher :param str offer: Marketplace image offer :param str sku: Marketplace image sku """ print('Creating pool [{}]...'.format(pool_id)) # Create a new pool of Linux compute nodes using an Azure Virtual Machines # Marketplace image. For more information about creating pools of Linux # nodes, see: # https://azure.microsoft.com/documentation/articles/batch-linux-nodes/ new_pool = batch.models.PoolAddParameter( id=pool_id, virtual_machine_configuration=batchmodels.VirtualMachineConfiguration( image_reference=batchmodels.ImageReference(publisher="Canonical", offer="UbuntuServer", sku="16.04-LTS", version="latest"), node_agent_sku_id="batch.node.ubuntu 16.04"), vm_size=config._POOL_VM_SIZE, target_dedicated_nodes=config._POOL_NODE_COUNT) batch_service_client.pool.add(new_pool)
def create_pool(pool_id, batch_service_client=None): """ Creates a pool of compute nodes. Parameters ========== pool_id: str, identifier for the pool batch_service_client: azure.batch.BatchServiceClient, A Batch service client. """ print("Creating pool [{}]...".format(pool_id)) if not batch_service_client: batch_service_client = create_batch_client() new_pool = batch.models.PoolAddParameter( id=pool_id, virtual_machine_configuration=batchmodels.VirtualMachineConfiguration( image_reference=batchmodels.ImageReference( publisher="Canonical", offer="UbuntuServer", sku="18.04-LTS", version="latest", ), node_agent_sku_id="batch.node.ubuntu 18.04", ), vm_size=config["pool_vm_size"], target_low_priority_nodes=config["pool_low_priority_node_count"], target_dedicated_nodes=config["pool_dedicated_node_count"], ) batch_service_client.pool.add(new_pool)
def create_pool_and_wait_for_nodes(batch_client, block_blob_client, pool_id, vm_size, vm_count): """Creates an Azure Batch pool with the specified id. :param batch_client: The batch client to use. :type batch_client: `batchserviceclient.BatchServiceClient` :param block_blob_client: The storage block blob client to use. :type block_blob_client: `azure.storage.blob.BlockBlobService` :param str pool_id: The id of the pool to create. :param str vm_size: vm size (sku) :param int vm_count: number of vms to allocate :rtype: list :return: list of `batchserviceclient.models.ComputeNode` """ # pick the latest supported 14.04 sku for UbuntuServer sku_to_use, image_ref_to_use = \ common.helpers.select_latest_verified_vm_image_with_node_agent_sku( batch_client, 'Canonical', 'UbuntuServer', '14.04') # upload start task script block_blob_client.create_container(_CONTAINER_NAME, fail_on_exist=False) sas_url = common.helpers.upload_blob_and_create_sas( block_blob_client, _CONTAINER_NAME, _STARTTASK_RESOURCE_FILE, _STARTTASK_SHELL_SCRIPT_PATH, datetime.datetime.utcnow() + datetime.timedelta(hours=1)) # create pool with start task pool = batchmodels.CloudPool( id=pool_id, enable_inter_node_communication=True, virtual_machine_configuration=batchmodels.VirtualMachineConfiguration( image_reference=image_ref_to_use, node_agent_sku_id=sku_to_use), vm_size=vm_size, target_dedicated=vm_count, start_task=batchmodels.StartTask( command_line=_STARTTASK_RESOURCE_FILE, run_elevated=True, wait_for_success=True, resource_files=[ batchmodels.ResourceFile(file_path=_STARTTASK_RESOURCE_FILE, blob_source=sas_url) ]), ) common.helpers.create_pool_if_not_exist(batch_client, pool) # because we want all nodes to be available before any tasks are assigned # to the pool, here we will wait for all compute nodes to reach idle nodes = common.helpers.wait_for_all_nodes_state( batch_client, pool, frozenset((batchmodels.ComputeNodeState.starttaskfailed, batchmodels.ComputeNodeState.unusable, batchmodels.ComputeNodeState.idle))) # ensure all node are idle if any(node.state != batchmodels.ComputeNodeState.idle for node in nodes): raise RuntimeError('node(s) of pool {} not in idle state'.format( pool.id)) return nodes
def create_pool(batch_service_client, pool_id, users): """ Creates a pool of compute nodes with the specified OS settings. :param batch_service_client: A Batch service client. :type batch_service_client: `azure.batch.BatchServiceClient` :param str pool_id: An ID for the new pool. :param str publisher: Marketplace image publisher :param str offer: Marketplace image offer :param str sku: Marketplace image sky """ print('Creating pool [{}]...'.format(pool_id)) # Create a new pool of Linux compute nodes using an Azure Virtual Machines # Marketplace image. For more information about creating pools of Linux # nodes, see: # https://azure.microsoft.com/documentation/articles/batch-linux-nodes/ # The start task installs ffmpeg on each node from an available repository, using # an administrator user identity. new_pool = batch.models.PoolAddParameter( id=pool_id, user_accounts=users, virtual_machine_configuration=batchmodels.VirtualMachineConfiguration( image_reference=batchmodels.ImageReference(publisher="Canonical", offer="UbuntuServer", sku="18.04-LTS", version="latest"), node_agent_sku_id="batch.node.ubuntu 18.04"), vm_size=_POOL_VM_SIZE, target_dedicated_nodes=_DEDICATED_POOL_NODE_COUNT, target_low_priority_nodes=_LOW_PRIORITY_POOL_NODE_COUNT, start_task=batchmodels.StartTask( #command_line="/bin/bash -c \"git clone https://github.com/uiuc-arc/probfuzz.git; \ # cd probfuzz/; ./install_java.sh; ./install.sh\"", # command_line="/bin/bash -c \"apt-get update\"", command_line="/bin/bash -c \"sudo apt-get -y update; \ sudo apt-get install git; \ sudo apt-get install -y python2.7; \ sudo apt-get install -y python-pip; \ sudo apt-get install -y bc; \ sudo apt-get install -y r-base; \ sudo pip2 --no-cache-dir install pandas; \ sudo pip2 install rpy2==2.8.6;\ sudo pip2 install argparse;\ sudo pip2 install numpy;\ sudo pip2 install scipy;\ \"", wait_for_success=True, user_identity=batchmodels.UserIdentity(user_name="admin"), #user_identity=batchmodels.UserIdentity( # auto_user=batchmodels.AutoUserSpecification( # scope=batchmodels.AutoUserScope.pool, # elevation_level=batchmodels.ElevationLevel.admin)), ), max_tasks_per_node=_MAX_TASKS_PER_NODE) batch_service_client.pool.add(new_pool)
def create_pool(batch_service_client, pool_id): """ Creates a pool of compute nodes with the specified OS settings. :param batch_service_client: A Batch service client. :type batch_service_client: `azure.batch.BatchServiceClient` :param str pool_id: An ID for the new pool. :param str publisher: Marketplace image publisher :param str offer: Marketplace image offer :param str sku: Marketplace image sky """ print('Creating pool [{}]...'.format(pool_id)) # Create a new pool of Linux compute nodes using an Azure Virtual Machines # Marketplace image. For more information about creating pools of Linux # nodes, see: # https://azure.microsoft.com/documentation/articles/batch-linux-nodes/ # The start task installs ffmpeg on each node from an available repository, using # an administrator user identity. # task_commands = [ "apt-get update", "apt-get -y install python3-pip", "apt -y install htop", "apt -y install iftop", "pip3 install azure-storage-blob", "pip3 install pyspark", "pip3 install pandas", "apt -y install openjdk-8-jre-headless" ] new_pool = batch.models.PoolAddParameter( id=pool_id, virtual_machine_configuration=batchmodels.VirtualMachineConfiguration( image_reference=batchmodels.ImageReference( publisher="Canonical", offer="UbuntuServer", sku="18.04-LTS", version="latest" ), node_agent_sku_id="batch.node.ubuntu 18.04"), vm_size=POOL_VM_SIZE, target_dedicated_nodes=DEDICATED_POOL_NODE_COUNT, target_low_priority_nodes=LOW_PRIORITY_POOL_NODE_COUNT, start_task=batchmodels.StartTask( command_line=wrap_commands_in_shell('linux',task_commands), wait_for_success=True, user_identity=batchmodels.UserIdentity( auto_user=batchmodels.AutoUserSpecification( scope=batchmodels.AutoUserScope.pool, elevation_level=batchmodels.ElevationLevel.admin)), ) ) batch_service_client.pool.add(new_pool)
def create_pool(batch_service_client, pool_id, resource_files, publisher, offer, sku, vm_size, node_count): """ Creates a pool of compute nodes with the specified OS settings. :param batch_service_client: A Batch service client. :type batch_service_client: `azure.batch.BatchServiceClient` :param str pool_id: An ID for the new pool. :param list resource_files: A collection of resource files for the pool's start task. :param str publisher: Marketplace image publisher :param str offer: Marketplace image offer :param str sku: Marketplace image sku """ print('Creating pool [{}]...'.format(pool_id)) # Create a new pool of Linux compute nodes using an Azure Virtual Machines # Marketplace image. For more information about creating pools of Linux # nodes, see: # https://azure.microsoft.com/documentation/articles/batch-linux-nodes/ # Specify the commands for the pool's start task. The start task is run # on each node as it joins the pool, and when it's rebooted or re-imaged. # If there were installs needed, this is where we'd add them as task_commands = ['echo starting up'] # Get the node agent SKU and image reference for the virtual machine # configuration. # For more information about the virtual machine configuration, see: # https://azure.microsoft.com/documentation/articles/batch-linux-nodes/ sku_to_use, image_ref_to_use = \ select_latest_verified_vm_image_with_node_agent_sku( batch_service_client, publisher, offer, sku) user = batchmodels.AutoUserSpecification( scope=batchmodels.AutoUserScope.pool, elevation_level=batchmodels.ElevationLevel.admin) new_pool = batch.models.PoolAddParameter( id=pool_id, virtual_machine_configuration=batchmodels.VirtualMachineConfiguration( image_reference=image_ref_to_use, node_agent_sku_id=sku_to_use), vm_size=vm_size, target_dedicated_nodes=node_count, start_task=batch.models.StartTask( command_line=wrap_commands_in_shell('linux',task_commands), user_identity=batchmodels.UserIdentity(auto_user=user), wait_for_success=True, resource_files=resource_files), ) try: batch_service_client.pool.add(new_pool) except batchmodels.batch_error.BatchErrorException as err: print(err) raise
def create_pool(self, pool_id): """ Creates a pool of compute nodes with the specified OS settings. :param str pool_id: An ID for the new pool. :param dict config: Configuration details. """ if pool_id in self.active_pools or self.pool_exists(pool_id): return self.logger.info("creating pool {}".format(pool_id)) pool_config = self.config['pools'][pool_id] sku_to_use, image_ref_to_use = self.__get_vm_image_and_node_agent_sku( pool_config) start_vm_commands = None if pool_config.get('create_vm_commands', None): start_vm_commands = self.__create_commands( pool_config['create_vm_commands']) user = batchmodels.AutoUserSpecification( scope=batchmodels.AutoUserScope.pool, elevation_level=batchmodels.ElevationLevel.admin) vm_configuration = batchmodels.VirtualMachineConfiguration( image_reference=image_ref_to_use, node_agent_sku_id=sku_to_use, ) vm_start_task = batchmodels.StartTask( command_line=self.__wrap_commands_in_shell('linux', start_vm_commands), user_identity=batchmodels.UserIdentity(auto_user=user), wait_for_success=True) new_pool = batchmodels.PoolAddParameter( id=pool_id, virtual_machine_configuration=vm_configuration, vm_size=pool_config['pool_vm_size'], enable_auto_scale=True, auto_scale_formula=pool_config['auto_scale_formula'], auto_scale_evaluation_interval=datetime.timedelta(minutes=5), start_task=vm_start_task, max_tasks_per_node=pool_config['max_tasks_per_node'], ) try: self.batch_client.pool.add(new_pool) except batchmodels.BatchErrorException as err: self.__print_batch_exception(err) raise self.active_pools.add(pool_id)
def test_batch_network_configuration(self, **kwargs): client = self.create_aad_client(**kwargs) # Test Create Pool with Network Config network_config = models.NetworkConfiguration( endpoint_configuration=models.PoolEndpointConfiguration( inbound_nat_pools=[ models.InboundNATPool( name="TestEndpointConfig", protocol=models.InboundEndpointProtocol.udp, backend_port=64444, frontend_port_range_start=60000, frontend_port_range_end=61000, network_security_group_rules=[ models.NetworkSecurityGroupRule( priority=150, access=models.NetworkSecurityGroupRuleAccess.allow, source_address_prefix='*' ) ] ) ] ) ) virtual_machine_config = models.VirtualMachineConfiguration( node_agent_sku_id="batch.node.ubuntu 16.04", image_reference=models.ImageReference( publisher="Canonical", offer="UbuntuServer", sku="16.04-LTS") ) pool = models.PoolAddParameter( id=self.get_resource_name('batch_network_'), target_dedicated_nodes=1, vm_size='Standard_A1', virtual_machine_configuration=virtual_machine_config, network_configuration=network_config ) client.pool.add(pool) network_pool = client.pool.get(pool.id) while self.is_live and network_pool.allocation_state != models.AllocationState.steady: time.sleep(10) network_pool = client.pool.get(pool.id) # Test Compute Node Config nodes = list(client.compute_node.list(pool.id)) self.assertEqual(len(nodes), 1) self.assertIsInstance(nodes[0], models.ComputeNode) self.assertEqual(len(nodes[0].endpoint_configuration.inbound_endpoints), 2) self.assertEqual(nodes[0].endpoint_configuration.inbound_endpoints[0].name, 'TestEndpointConfig.0') self.assertEqual(nodes[0].endpoint_configuration.inbound_endpoints[0].protocol.value, 'udp')
def create_pool(batch_service_client, pool_id): """ Creates a pool of compute nodes with the specified OS settings. :param batch_service_client: A Batch service client. :type batch_service_client: `azure.batch.BatchServiceClient` :param str pool_id: An ID for the new pool. :param str publisher: Marketplace image publisher :param str offer: Marketplace image offer :param str sku: Marketplace image sku """ print('Creating pool [{}]...'.format(pool_id)) # Create a new pool of Linux compute nodes using an Azure Virtual Machines # Marketplace image. For more information about creating pools of Linux # nodes, see: # https://azure.microsoft.com/documentation/articles/batch-linux-nodes/ # In this case is a pool ready to run Docker containers image_ref_to_use = batch.models.ImageReference( publisher='microsoft-azure-batch', offer='ubuntu-server-container', sku='16-04-lts', version='latest' ) # Specify a container registry # We got the credentials from config.py containerRegistry = batchmodels.ContainerRegistry( user_name=config._REGISTRY_USER_NAME, password=config._REGISTRY_PASSWORD, registry_server=config._REGISTRY_SERVER ) # The instance will pull the images defined here container_conf = batchmodels.ContainerConfiguration( container_image_names=[config._DOCKER_IMAGE], container_registries=[containerRegistry] ) new_pool = batch.models.PoolAddParameter( id=pool_id, virtual_machine_configuration=batchmodels.VirtualMachineConfiguration( image_reference=image_ref_to_use, container_configuration=container_conf, node_agent_sku_id='batch.node.ubuntu 16.04'), vm_size=config._POOL_VM_SIZE, target_dedicated_nodes=config._POOL_NODE_COUNT ) batch_service_client.pool.add(new_pool)
def get_virtual_machine_configuration(self): if self.image_id: image_ref = batchmodels.ImageReference( virtual_machine_image_id=self.image_id) else: image_ref = batchmodels.ImageReference( publisher=self.image_publisher, offer=self.image_offer, sku=self.image_sku, version=self.image_version) return batchmodels.VirtualMachineConfiguration( image_reference=image_ref, node_agent_sku_id=self.node_agent_sku_id)
def create_pool_and_wait_for_node(batch_client, pool_id, vm_size, vm_count, sha1_cert_tp): """Creates an Azure Batch pool with the specified id. :param batch_client: The batch client to use. :type batch_client: `batchserviceclient.BatchServiceClient` :param str pool_id: The id of the pool to create. :param str vm_size: vm size (sku) :param int vm_count: number of vms to allocate :param str sha1_cert_tp: sha1 cert thumbprint for cert ref """ # pick the latest supported 16.04 sku for UbuntuServer sku_to_use, image_ref_to_use = \ common.helpers.select_latest_verified_vm_image_with_node_agent_sku( batch_client, 'Canonical', 'UbuntuServer', '16.04') # create start task commands # 1. update repository # 2. install blobxfer pre-requisites # 3. pip install blobxfer python script start_task_commands = [ 'apt-get update', 'apt-get install -y build-essential libssl-dev libffi-dev ' + 'libpython-dev python-dev python-pip', 'pip install --upgrade blobxfer' ] user = batchmodels.AutoUserSpecification( scope=batchmodels.AutoUserScope.pool, elevation_level=batchmodels.ElevationLevel.admin) # create pool with start task and cert ref with visibility of task pool = batchmodels.PoolAddParameter( id=pool_id, virtual_machine_configuration=batchmodels.VirtualMachineConfiguration( image_reference=image_ref_to_use, node_agent_sku_id=sku_to_use), vm_size=vm_size, target_dedicated=vm_count, start_task=batchmodels.StartTask( command_line=common.helpers.wrap_commands_in_shell( 'linux', start_task_commands), user_identity=batchmodels.UserIdentity(auto_user=user), wait_for_success=True), certificate_references=[ batchmodels.CertificateReference( sha1_cert_tp, 'sha1', visibility=[batchmodels.CertificateVisibility.task]) ], ) common.helpers.create_pool_if_not_exist(batch_client, pool)
def create_processing_pool(batch_service_client, start_task): """ Creates a pool of compute nodes with the specified OS settings. :param batch_service_client: A Batch service client. :param str start_task: task start command. :type batch_service_client: `azure.batch.BatchServiceClient` """ LOGGER.info(f'Creating pool [{PROCESSING_POOL_ID}]...') image_ref_to_use = get_image_reference() container_registry = \ batch_models.ContainerRegistry( registry_server=REGISTRY_SERVER, user_name=REGISTRY_ACCOUNT_USER, password=REGISTRY_ACCOUNT_PASSWORD) container_conf = batch_models.ContainerConfiguration( container_image_names=[DOCKER_CONTAINER_URL], container_registries=[container_registry]) new_pool = batch_models.PoolAddParameter( id=PROCESSING_POOL_ID, virtual_machine_configuration= batch_models.VirtualMachineConfiguration( image_reference=image_ref_to_use, container_configuration=container_conf, node_agent_sku_id=VM_AGENT_SKU), vm_size=PROCESSING_POOL_VM_SIZE, start_task=batch_models.StartTask( command_line=start_task, user_identity=batch_models.UserIdentity( auto_user=batch_models.AutoUserSpecification( scope='pool', elevation_level='admin')) ), enable_auto_scale=True, auto_scale_evaluation_interval=datetime.timedelta( minutes=PROCESSING_POOL_SCALE_INTERVAL_MINUTES), auto_scale_formula=PROCESSING_POOL_SCALE_FORMULA) try: batch_service_client.pool.add(new_pool) LOGGER.info("Processing Pool Created") except batch_models.BatchErrorException as err: if 'The specified pool already exists.' in err.error.message.value: LOGGER.info("Pool already exists...") else: raise
def __create_pool_and_job(self, cluster_conf, software_metadata_key: str, start_task, VmImageModel): """ Create a pool and job :param cluster_conf: the configuration object used to create the cluster :type cluster_conf: aztk.models.ClusterConfiguration :parm software_metadata_key: the id of the software being used on the cluster :param start_task: the start task for the cluster :param VmImageModel: the type of image to provision for the cluster :param wait: wait until the cluster is ready """ # reuse pool_id as job_id pool_id = cluster_conf.cluster_id job_id = cluster_conf.cluster_id # Get a verified node agent sku sku_to_use, image_ref_to_use = \ helpers.select_latest_verified_vm_image_with_node_agent_sku( VmImageModel.publisher, VmImageModel.offer, VmImageModel.sku, self.batch_client) # Confiure the pool pool = batch_models.PoolAddParameter( id=pool_id, virtual_machine_configuration=batch_models.VirtualMachineConfiguration( image_reference=image_ref_to_use, node_agent_sku_id=sku_to_use), vm_size=cluster_conf.vm_size, target_dedicated_nodes=cluster_conf.vm_count, target_low_priority_nodes=cluster_conf.vm_low_pri_count, start_task=start_task, enable_inter_node_communication=True, max_tasks_per_node=1, metadata=[ batch_models.MetadataItem( name=constants.AZTK_SOFTWARE_METADATA_KEY, value=software_metadata_key), ]) # Create the pool + create user for the pool helpers.create_pool_if_not_exist(pool, self.batch_client) # Create job job = batch_models.JobAddParameter( id=job_id, pool_info=batch_models.PoolInformation(pool_id=pool_id)) # Add job to batch self.batch_client.job.add(job) return helpers.get_cluster(cluster_conf.cluster_id, self.batch_client)
def create_pool(config, batch_service_client): """ Creates a pool of compute nodes with the specified OS settings. :param batch_service_client: A Batch service client. :type batch_service_client: `azure.batch.BatchServiceClient` :param str pool_id: An ID for the new pool. :param str publisher: Marketplace image publisher :param str offer: Marketplace image offer :param str sku: Marketplace image sku """ # Create a new pool of Linux compute nodes using an Azure Virtual Machines # Marketplace image. For more information about creating pools of Linux # nodes, see: # https://azure.microsoft.com/documentation/articles/batch-linux-nodes/ image_ref_to_use = models.ImageReference( publisher="microsoft-azure-batch", offer="ubuntu-server-container", sku="16-04-lts", version="latest", ) if config.REGISTRY_USERNAME: registry = models.ContainerRegistry( user_name=config.REGISTRY_USERNAME, password=config.REGISTRY_PASSWORD, registry_server=config.REGISTRY_SERVER, ) container_conf = models.ContainerConfiguration( container_image_names=[config.DOCKER_CONTAINER], container_registries=[registry], ) else: container_conf = models.ContainerConfiguration( container_image_names=[config.DOCKER_CONTAINER]) new_pool = models.PoolAddParameter( id=config.POOL_ID, virtual_machine_configuration=models.VirtualMachineConfiguration( image_reference=image_ref_to_use, container_configuration=container_conf, node_agent_sku_id="batch.node.ubuntu 16.04", ), vm_size=config.POOL_VM_SIZE, target_dedicated_nodes=config.POOL_NODE_COUNT, target_low_priority_nodes=config.POOL_LOW_PRIORITY_NODE_COUNT, ) batch_service_client.pool.add(new_pool)
def create_pool(batch_service_client, pool_id): """ Creates a pool of compute nodes with the specified OS settings. :param batch_service_client: A Batch service client. :type batch_service_client: `azure.batch.BatchServiceClient` :param str pool_id: An ID for the new pool. :param str publisher: Marketplace image publisher :param str offer: Marketplace image offer :param str sku: Marketplace image sky """ print('Creating pool [{}]...'.format(pool_id)) # Create a new pool of Linux compute nodes using an Azure Virtual Machines # Marketplace image. For more information about creating pools of Linux # nodes, see: # https://azure.microsoft.com/documentation/articles/batch-linux-nodes/ # The start task installs ffmpeg on each node from an available repository, using # an administrator user identity. new_pool = batch.models.PoolAddParameter( id=pool_id, virtual_machine_configuration=batchmodels.VirtualMachineConfiguration( image_reference=batchmodels.ImageReference(publisher="Canonical", offer="UbuntuServer", sku="16.04-LTS", version="latest"), node_agent_sku_id="batch.node.ubuntu 16.04"), vm_size=os.environ['POOL_VM_SIZE'], target_dedicated_nodes=os.environ['DEDICATED_POOL_NODE_COUNT'], target_low_priority_nodes=os.environ['LOW_PRIORITY_POOL_NODE_COUNT'], start_task=batchmodels.StartTask( command_line="/bin/bash -c \"apt-get update && \ apt-get install -y apt-transport-https ca-certificates curl software-properties-common &&\ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \ add-apt-repository 'deb [arch=amd64] https://download.docker.com/linux/ubuntu xenial stable' && \ apt-get update && \ apt-get install -y docker-ce\"", wait_for_success=True, user_identity=batchmodels.UserIdentity( auto_user=batchmodels.AutoUserSpecification( scope=batchmodels.AutoUserScope.pool, elevation_level=batchmodels.ElevationLevel.admin)), )) batch_service_client.pool.add(new_pool)
def create_pool(batch_service_client, pool_id, publisher, offer, sku, pool_vm_size, pool_node_count, low_priority_node_count, task_commands): """ Creates a pool of compute nodes with the specified OS settings. :param batch_service_client: A Batch service client. :type batch_service_client: `azure.batch.BatchServiceClient` :param str pool_id: An ID for the new pool. :param list resource_files: A collection of resource files for the pool's start task. :param str publisher: Marketplace image publisher :param str offer: Marketplace image offer :param str sku: Marketplace image sku """ print('Creating pool [{}]...'.format(pool_id)) # Get the node agent SKU and image reference for the virtual machine # configuration. # For more information about the virtual machine configuration, see: # https://azure.microsoft.com/documentation/articles/batch-linux-nodes/ sku_to_use, image_ref_to_use = \ common.helpers.select_latest_verified_vm_image_with_node_agent_sku( batch_service_client, publisher, offer, sku) user = batchmodels.AutoUserSpecification( scope=batchmodels.AutoUserScope.pool, elevation_level=batchmodels.ElevationLevel.admin) new_pool = batch.models.PoolAddParameter( id=pool_id, virtual_machine_configuration=batchmodels.VirtualMachineConfiguration( image_reference=image_ref_to_use, node_agent_sku_id=sku_to_use), vm_size=pool_vm_size, target_dedicated_nodes=pool_node_count, target_low_priority_nodes=low_priority_node_count, start_task=batch.models.StartTask( command_line=common.helpers.wrap_commands_in_shell('linux', task_commands), user_identity=batchmodels.UserIdentity(auto_user=user), wait_for_success=True, max_task_retry_count=-1) ) try: batch_service_client.pool.add(new_pool) except batchmodels.batch_error.BatchErrorException as err: print_batch_exception(err) raise
def get_vm_config_for_distro(batch_service_client, distro, version): """ Gets a virtual machine configuration for the specified distro and version from the list of Azure Virtual Machines Marketplace images verified to be compatible with the Batch service. :param batch_service_client: A Batch service client. :type batch_service_client: `azure.batch.BatchServiceClient` :param str distro: The Linux distribution that should be installed on the compute nodes, e.g. 'Ubuntu' or 'CentOS'. Supports partial string matching. :param str version: The version of the operating system for the compute nodes, e.g. '15' or '14.04'. Supports partial string matching. :rtype: `azure.batch.models.VirtualMachineConfiguration` :return: A virtual machine configuration specifying the Virtual Machines Marketplace image and node agent SKU to install on the compute nodes in a pool. """ # Get the list of node agents from the Batch service node_agent_skus = batch_service_client.account.list_node_agent_skus() # Get the first node agent that is compatible with the specified distro. # Note that 'distro' in this case actually maps to the 'offer' property of # the ImageReference. node_agent = next(agent for agent in node_agent_skus for image_ref in agent.verified_image_references if distro.lower() in image_ref.offer.lower() and version.lower() in image_ref.sku.lower()) # Get the last image reference from the list of verified references # for the node agent we obtained. Typically, the verified image # references are returned in ascending release order so this should give us # the newest image. img_ref = [ image_ref for image_ref in node_agent.verified_image_references if distro.lower() in image_ref.offer.lower() and version.lower() in image_ref.sku.lower() ][-1] # Create the VirtualMachineConfiguration, specifying the VM image # reference and the Batch node agent to be installed on the node. # Note that these commands are valid for a pool of Ubuntu-based compute # nodes, and that you may need to adjust the commands for execution # on other distros. vm_config = batchmodels.VirtualMachineConfiguration( image_reference=img_ref, node_agent_sku_id=node_agent.id) return vm_config
def create_pool_and_wait_for_nodes(batch_client, block_blob_client, pool_id, vm_size, vm_count): sku_to_use, image_ref_to_use = common.helpers.select_latest_verified_vm_image_with_node_agent_sku( batch_client, 'Canonical', 'UbuntuServer', '14.04') block_blob_client.create_container(_CONTAINER_NAME, fail_on_exist=False) # upload start task script block_blob_client.create_container(_CONTAINER_NAME, fail_on_exist=False) sas_url = common.helpers.upload_blob_and_create_sas( block_blob_client, _CONTAINER_NAME, _STARTTASK_RESOURCE_FILE, _STARTTASK_SHELL_SCRIPT_PATH, datetime.datetime.utcnow() + datetime.timedelta(hours=1)) # create pool and execute starttask pool = batchmodels.PoolAddParameter( id=pool_id, enable_inter_node_communication=True, virtual_machine_configuration=batchmodels.VirtualMachineConfiguration( image_reference=image_ref_to_use, node_agent_sku_id=sku_to_use), vm_size=vm_size, target_dedicated=vm_count, start_task=batchmodels.StartTask( command_line=_STARTTASK_RESOURCE_FILE, run_elevated=True, wait_for_success=True, resource_files=[ batchmodels.ResourceFile(file_path=_STARTTASK_RESOURCE_FILE, blob_source=sas_url) ]), ) common.helpers.create_pool_if_not_exist(batch_client, pool) # because we want all nodes to be available before any tasks are assigned # to the pool, here we will wait for all compute nodes to reach idle nodes = common.helpers.wait_for_all_nodes_state( batch_client, pool, frozenset((batchmodels.ComputeNodeState.starttaskfailed, batchmodels.ComputeNodeState.unusable, batchmodels.ComputeNodeState.idle))) # ensure all node are idle if any(node.state != batchmodels.ComputeNodeState.idle for node in nodes): raise RuntimeError('node(s) of pool {} not in idle state'.format( pool.id)) return nodes
def create_pool(batch_service_client, pool_id, application_files): """ Creates a pool of compute nodes with the specified OS settings. :param batch_service_client: A Batch service client. :type batch_service_client: `azure.batch.BatchServiceClient` :param str pool_id: An ID for the new pool. :param str publisher: Marketplace image publisher :param str offer: Marketplace image offer :param str sku: Marketplace image sky """ print('Creating pool [{}]...'.format(pool_id)) # Create a new pool of Linux compute nodes using an Azure Virtual Machines # Marketplace image. For more information about creating pools of Linux # nodes, see: # https://azure.microsoft.com/documentation/articles/batch-linux-nodes/ # The start task installs ffmpeg on each node from an available repository, using # an administrator user identity. new_pool = batch.models.PoolAddParameter( id=pool_id, virtual_machine_configuration=batchmodels.VirtualMachineConfiguration( image_reference=batchmodels.ImageReference( publisher="MicrosoftWindowsServer", offer="WindowsServer", sku="2016-Datacenter", version="latest" ), node_agent_sku_id="batch.node.windows amd64"), vm_size=config._POOL_VM_SIZE, target_dedicated_nodes=config._DEDICATED_POOL_NODE_COUNT, target_low_priority_nodes=config._LOW_PRIORITY_POOL_NODE_COUNT, start_task=batchmodels.StartTask( command_line="cmd /c starttask.cmd", resource_files=application_files, wait_for_success=True, user_identity=batchmodels.UserIdentity( auto_user=batchmodels.AutoUserSpecification( scope=batchmodels.AutoUserScope.pool, elevation_level=batchmodels.ElevationLevel.admin)), ) ) batch_service_client.pool.add(new_pool)
def create_pool(batch_service_client, pool_id, resource_files): """ Creates a pool of compute nodes with the specified OS settings. :param batch_service_client: A Batch service client. :type batch_service_client: `azure.batch.BatchServiceClient` :param str pool_id: An ID for the new pool. :param str publisher: Marketplace image publisher :param str offer: Marketplace image offer :param str sku: Marketplace image sku """ print('Creating pool [{}]...'.format(pool_id)) # Create a new pool of Linux compute nodes using an Azure Virtual Machines # Marketplace image. For more information about creating pools of Linux # nodes, see: # https://azure.microsoft.com/documentation/articles/batch-linux-nodes/ task_commands = [ 'cp -p {} $AZ_BATCH_NODE_SHARED_DIR'.format(config._TASK_FILE), '''wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh && \ bash miniconda.sh -b -p $AZ_BATCH_NODE_SHARED_DIR/miniconda && \ export PATH="$AZ_BATCH_NODE_SHARED_DIR/miniconda/bin:$PATH" && \ source "$AZ_BATCH_NODE_SHARED_DIR/miniconda/bin/activate" && \ conda install -y -c anaconda -c conda-forge \ pip''' ] image_ref = batchmodels.ImageReference(publisher="Canonical", offer="UbuntuServer", sku="18.04-LTS", version="latest") vm_config = batchmodels.VirtualMachineConfiguration( image_reference=image_ref, node_agent_sku_id="batch.node.ubuntu 18.04") new_pool = batch.models.PoolAddParameter( id=pool_id, virtual_machine_configuration=vm_config, vm_size=config._POOL_VM_SIZE, target_low_priority_nodes=config._POOL_NODE_COUNT, start_task=batch.models.StartTask(command_line=wrap_commands_in_shell( 'linux', task_commands), resource_files=resource_files)) batch_service_client.pool.add(new_pool)
def create_pool(self): from azure.batch import models as batchmodels pool_id = self.compute_env_prefix + self.machine.name + '-' + str( self.disk_size) pool = self.get_pool(pool_id) if pool is not None: return pool_id sku_to_use, image_ref_to_use = self.select_latest_verified_vm_image_with_node_agent_sku( ) container_configuration = batchmodels.ContainerConfiguration( container_image_names=[self.image]) config = batchmodels.VirtualMachineConfiguration( image_reference=image_ref_to_use, node_agent_sku_id=sku_to_use, data_disks=[ batchmodels.DataDisk(disk_size_gb=self.disk_size, lun=1) ], container_configuration=container_configuration, ) pool = batchmodels.PoolAddParameter( id=pool_id, display_name=pool_id, virtual_machine_configuration=config, vm_size=self.machine.name, ) if self.conf[utils.PLATFORM].get('low_priority', False): pool.target_low_priority_nodes = 1 else: pool.target_dedicated_nodes = 1 self.batch_client.pool.add(pool) while self.get_pool(pool_id) is None: time.sleep(1) return pool_id