def create_pool(config, batch_service_client):
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str pool_id: An ID for the new pool.
:param str publisher: Marketplace image publisher
:param str offer: Marketplace image offer
:param str sku: Marketplace image sku
"""
# Create a new pool of Linux compute nodes using an Azure Virtual Machines
# Marketplace image. For more information about creating pools of Linux
# nodes, see:
# https://azure.microsoft.com/documentation/articles/batch-linux-nodes/
image_ref_to_use = models.ImageReference(
publisher="microsoft-azure-batch",
offer="ubuntu-server-container",
sku="16-04-lts",
version="latest",
)
if config.REGISTRY_USERNAME:
registry = models.ContainerRegistry(
user_name=config.REGISTRY_USERNAME,
password=config.REGISTRY_PASSWORD,
registry_server=config.REGISTRY_SERVER,
)
container_conf = models.ContainerConfiguration(
container_image_names=[config.DOCKER_CONTAINER],
container_registries=[registry],
)
else:
container_conf = models.ContainerConfiguration(
container_image_names=[config.DOCKER_CONTAINER])
new_pool = models.PoolAddParameter(
id=config.POOL_ID,
virtual_machine_configuration=models.VirtualMachineConfiguration(
image_reference=image_ref_to_use,
container_configuration=container_conf,
node_agent_sku_id="batch.node.ubuntu 16.04",
),
vm_size=config.POOL_VM_SIZE,
target_dedicated_nodes=config.POOL_NODE_COUNT,
target_low_priority_nodes=config.POOL_LOW_PRIORITY_NODE_COUNT,
)
batch_service_client.pool.add(new_pool)
def create_container_config(config: Dict[str, str]) -> batchmodels.ContainerConfiguration:
ada_cr = batchmodels.ContainerRegistry(
registry_server="ada510.azurecr.io",
user_name="ada510",
password=config["CR_PASSWORD"],
)
return batchmodels.ContainerConfiguration(
container_image_names=[NEO_IMAGE],
container_registries=[ada_cr],
)
def _create_pool(self):
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str pool_id: An ID for the new pool.
:param str publisher: Marketplace image publisher
:param str offer: Marketplace image offer
:param str sku: Marketplace image sku
"""
if self.config.REGISTRY_SERVER:
print("Using a private registry")
registry = models.ContainerRegistry(
user_name=self.config.REGISTRY_USERNAME,
password=self.config.REGISTRY_PASSWORD,
registry_server=self.config.REGISTRY_SERVER,
)
container_conf = models.ContainerConfiguration(
container_image_names=[self.config.DOCKER_IMAGE],
container_registries=[registry],
)
else:
container_conf = models.ContainerConfiguration(
container_image_names=[self.config.DOCKER_IMAGE])
new_pool = models.PoolAddParameter(
id=self.config.POOL_ID,
virtual_machine_configuration=models.VirtualMachineConfiguration(
image_reference=_IMAGE_REF,
container_configuration=container_conf,
node_agent_sku_id="batch.node.ubuntu 16.04",
),
vm_size=self.config.POOL_VM_SIZE,
target_dedicated_nodes=self.config.POOL_NODE_COUNT,
target_low_priority_nodes=self.config.POOL_LOW_PRIORITY_NODE_COUNT,
)
# Create the pool
self.batch_client.pool.add(new_pool)
def create_pool(batch_service_client, pool_id):
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str pool_id: An ID for the new pool.
:param str publisher: Marketplace image publisher
:param str offer: Marketplace image offer
:param str sku: Marketplace image sku
"""
print('Creating pool [{}]...'.format(pool_id))
# Create a new pool of Linux compute nodes using an Azure Virtual Machines
# Marketplace image. For more information about creating pools of Linux
# nodes, see:
# https://azure.microsoft.com/documentation/articles/batch-linux-nodes/
# In this case is a pool ready to run Docker containers
image_ref_to_use = batch.models.ImageReference(
publisher='microsoft-azure-batch',
offer='ubuntu-server-container',
sku='16-04-lts',
version='latest'
)
# Specify a container registry
# We got the credentials from config.py
containerRegistry = batchmodels.ContainerRegistry(
user_name=config._REGISTRY_USER_NAME,
password=config._REGISTRY_PASSWORD,
registry_server=config._REGISTRY_SERVER
)
# The instance will pull the images defined here
container_conf = batchmodels.ContainerConfiguration(
container_image_names=[config._DOCKER_IMAGE],
container_registries=[containerRegistry]
)
new_pool = batch.models.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=image_ref_to_use,
container_configuration=container_conf,
node_agent_sku_id='batch.node.ubuntu 16.04'),
vm_size=config._POOL_VM_SIZE,
target_dedicated_nodes=config._POOL_NODE_COUNT
)
batch_service_client.pool.add(new_pool)
def create_processing_pool(batch_service_client, start_task):
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:param str start_task: task start command.
:type batch_service_client: `azure.batch.BatchServiceClient`
"""
LOGGER.info(f'Creating pool [{PROCESSING_POOL_ID}]...')
image_ref_to_use = get_image_reference()
container_registry = \
batch_models.ContainerRegistry(
registry_server=REGISTRY_SERVER,
user_name=REGISTRY_ACCOUNT_USER,
password=REGISTRY_ACCOUNT_PASSWORD)
container_conf = batch_models.ContainerConfiguration(
container_image_names=[DOCKER_CONTAINER_URL],
container_registries=[container_registry])
new_pool = batch_models.PoolAddParameter(
id=PROCESSING_POOL_ID,
virtual_machine_configuration=
batch_models.VirtualMachineConfiguration(
image_reference=image_ref_to_use,
container_configuration=container_conf,
node_agent_sku_id=VM_AGENT_SKU),
vm_size=PROCESSING_POOL_VM_SIZE,
start_task=batch_models.StartTask(
command_line=start_task,
user_identity=batch_models.UserIdentity(
auto_user=batch_models.AutoUserSpecification(
scope='pool',
elevation_level='admin'))
),
enable_auto_scale=True,
auto_scale_evaluation_interval=datetime.timedelta(
minutes=PROCESSING_POOL_SCALE_INTERVAL_MINUTES),
auto_scale_formula=PROCESSING_POOL_SCALE_FORMULA)
try:
batch_service_client.pool.add(new_pool)
LOGGER.info("Processing Pool Created")
except batch_models.BatchErrorException as err:
if 'The specified pool already exists.' in err.error.message.value:
LOGGER.info("Pool already exists...")
else:
raise
def create_pool(self):
from azure.batch import models as batchmodels
pool_id = self.compute_env_prefix + self.machine.name + '-' + str(
self.disk_size)
pool = self.get_pool(pool_id)
if pool is not None:
return pool_id
sku_to_use, image_ref_to_use = self.select_latest_verified_vm_image_with_node_agent_sku(
)
container_configuration = batchmodels.ContainerConfiguration(
container_image_names=[self.image])
config = batchmodels.VirtualMachineConfiguration(
image_reference=image_ref_to_use,
node_agent_sku_id=sku_to_use,
data_disks=[
batchmodels.DataDisk(disk_size_gb=self.disk_size, lun=1)
],
container_configuration=container_configuration,
)
pool = batchmodels.PoolAddParameter(
id=pool_id,
display_name=pool_id,
virtual_machine_configuration=config,
vm_size=self.machine.name,
)
if self.conf[utils.PLATFORM].get('low_priority', False):
pool.target_low_priority_nodes = 1
else:
pool.target_dedicated_nodes = 1
self.batch_client.pool.add(pool)
while self.get_pool(pool_id) is None:
time.sleep(1)
return pool_id
def create_pool_with_containers(batch_service_client, pool_id, resource_files,
publisher, offer, sku):
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str pool_id: An ID for the new pool.
:param list resource_files: A collection of resource files for the pool's
start task.
:param str publisher: Marketplace image publisher
:param str offer: Marketplace image offer
:param str sku: Marketplace image sku
"""
print('Creating pool [{}]...'.format(pool_id))
# Create a new pool of Linux compute nodes using an Azure Virtual Machines
# Marketplace image. For more information about creating pools of Linux
# nodes, see:
# https://azure.microsoft.com/documentation/articles/batch-linux-nodes/
# Specify the commands for the pool's start task. The start task is run
# on each node as it joins the pool, and when it's rebooted or re-imaged.
# We use the start task to prep the node for running our task script.
task_commands = [
# Copy the python_tutorial_task.py script to the "shared" directory
# that all tasks that run on the node have access to. Note that
# we are using the -p flag with cp to preserve the file uid/gid,
# otherwise since this start task is run as an admin, it would not
# be accessible by tasks run as a non-admin user.
#'wget https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb',
#'sudo dpkg -i packages-microsoft-prod.deb',
#'wget -O azcopy.tar.gz https://aka.ms/downloadazcopylinux64',
#'tar -xf azcopy.tar.gz',
#'sudo ./install.sh',
#'wget https://repo.anaconda.com/archive/Anaconda3-5.1.0-Linux-x86_64.sh -O ~/conda.sh',
#'bash ~/conda.sh -b -p $AZ_BATCH_NODE_SHARED_DIR/conda',
#'export PATH="$AZ_BATCH_NODE_SHARED_DIR/conda/bin:$PATH"',
#'sudo apt-get -y update',
#'sudo apt-get -y install azcopy',
'cp -p {} $AZ_BATCH_NODE_SHARED_DIR'.format(_TUTORIAL_TASK_FILE),
#'cp -p {} $AZ_BATCH_NODE_SHARED_DIR'.format(_ENV_YML_FILE),
'azcopy --source https://{0}.blob.core.windows.net/model/ghanamines.h5 --destination $AZ_BATCH_NODE_SHARED_DIR/ghanamines.h5 --source-key {1}'
.format(_STORAGE_ACCOUNT_NAME, _STORAGE_ACCOUNT_KEY),
#'sudo $AZ_BATCH_NODE_SHARED_DIR/conda/bin/conda env create -f {}'.format(_ENV_YML_FILE)
]
# Get the node agent SKU and image reference for the virtual machine
# configuration.
# For more information about the virtual machine configuration, see:
# https://azure.microsoft.com/documentation/articles/batch-linux-nodes/
sku_to_use, image_ref_to_use = \
common_helpers.select_latest_verified_vm_image_with_node_agent_sku(
batch_service_client, publisher, offer, sku)
user = batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)
container_reg = batchmodels.ContainerRegistry(user_name=CLIENT_ID,
password=SECRET,
registry_server=_ACR_URL)
container_cfg = batchmodels.ContainerConfiguration(
container_image_names=[_ACR_IMG_NAME],
container_registries=[container_reg])
my_img_ref = batchmodels.ImageReference(
virtual_machine_image_id=_CUSTOM_VM_IMG_ID)
vm_cfg = batchmodels.VirtualMachineConfiguration(
image_reference=my_img_ref,
node_agent_sku_id=
sku_to_use, #'batch.node.ubuntu 16.04', ##verificare che l'immagine ghanaimg abbia gpu
container_configuration=container_cfg)
task_containersettings = batchmodels.TaskContainerSettings(
image_name=_ACR_IMG_NAME)
new_pool = batchmodels.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=vm_cfg,
vm_size=_POOL_VM_SIZE,
target_dedicated_nodes=_POOL_NODE_COUNT,
target_low_priority_nodes=1,
start_task=batch.models.StartTask(
command_line=common_helpers.wrap_commands_in_shell(
'linux', task_commands),
user_identity=batchmodels.UserIdentity(auto_user=user),
wait_for_success=True,
resource_files=resource_files,
container_settings=task_containersettings))
try:
batch_service_client.pool.add(new_pool)
except batchmodels.batch_error.BatchErrorException as err:
print_batch_exception(err)
raise
