def create_pool(batch_client, pool_id, vm_size, vm_count, app_files):
"""Creates an Azure Batch pool with the specified id.
:param batch_client: The batch client to use.
:type batch_client: `batchserviceclient.BatchServiceClient`
:param block_blob_client: The storage block blob client to use.
:type block_blob_client: `azure.storage.blob.BlockBlobService`
:param str pool_id: The id of the pool to create.
:param str vm_size: vm size (sku)
:param int vm_count: number of vms to allocate
:param list app_files: The list of all the other scripts to upload.
"""
# pick the latest supported 16.04 sku for UbuntuServer
sku_to_use, image_ref_to_use = \
common.helpers.select_latest_verified_vm_image_with_node_agent_sku(
batch_client, 'Canonical', 'UbuntuServer', '14.04')
user = batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)
task_commands = get_list_from_file('configs/start_commands')
print(task_commands)
pool = batchmodels.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=image_ref_to_use, node_agent_sku_id=sku_to_use),
vm_size=vm_size,
target_dedicated=vm_count,
start_task=batchmodels.StartTask(
command_line=common.helpers.wrap_commands_in_shell(
'linux', task_commands),
user_identity=batchmodels.UserIdentity(auto_user=user),
resource_files=app_files,
wait_for_success=True))
common.helpers.create_pool_if_not_exist(batch_client, pool)
def pool_create():
image_reference = batchmodel.ImageReference(
publisher=config_azure['batch_pool_image_publisher'],
offer=config_azure['batch_pool_image_offer'],
sku=config_azure['batch_pool_image_sku'])
vm_config = batchmodel.VirtualMachineConfiguration(
image_reference=image_reference,
node_agent_sku_id=config_azure['batch_pool_node_agent_sku'])
vm_start_task = batchmodel.StartTask(
command_line=
'/bin/bash -c "sudo yum -y install epel-release; sudo yum -y install python36 python36-devel python36-tools; sudo python36 -m ensurepip; sudo yum -y install openmpi openmpi-devel; sudo env MPICC=/usr/lib64/openmpi/bin/mpicc pip3 install mpi4py numpy; sudo pip3 --yes uninstall azure azure-common azure-storage; sudo pip3 install azure-storage azure-batch"',
user_identity=batchmodel.UserIdentity(
auto_user=batchmodel.AutoUserSpecification(
scope=batchmodel.AutoUserScope.pool,
elevation_level=batchmodel.ElevationLevel.admin)),
wait_for_success=True)
batch_service.pool.add(pool=batchmodel.PoolAddParameter(
id=config_azure['batch_pool_name'],
vm_size=config_azure['batch_pool_vm_size'],
virtual_machine_configuration=vm_config,
target_dedicated_nodes=config_azure[
'batch_pool_target_dedicated_nodes'],
enable_inter_node_communication=True,
start_task=vm_start_task),
raw=True)
def create_pool_and_wait_for_nodes(batch_client, block_blob_client, pool_id,
vm_size, vm_count):
"""Creates an Azure Batch pool with the specified id.
:param batch_client: The batch client to use.
:type batch_client: `batchserviceclient.BatchServiceClient`
:param block_blob_client: The storage block blob client to use.
:type block_blob_client: `azure.storage.blob.BlockBlobService`
:param str pool_id: The id of the pool to create.
:param str vm_size: vm size (sku)
:param int vm_count: number of vms to allocate
:rtype: list
:return: list of `batchserviceclient.models.ComputeNode`
"""
# pick the latest supported 14.04 sku for UbuntuServer
sku_to_use, image_ref_to_use = \
common.helpers.select_latest_verified_vm_image_with_node_agent_sku(
batch_client, 'Canonical', 'UbuntuServer', '14.04')
# upload start task script
block_blob_client.create_container(_CONTAINER_NAME, fail_on_exist=False)
sas_url = common.helpers.upload_blob_and_create_sas(
block_blob_client, _CONTAINER_NAME, _STARTTASK_RESOURCE_FILE,
_STARTTASK_SHELL_SCRIPT_PATH,
datetime.datetime.utcnow() + datetime.timedelta(hours=1))
# create pool with start task
pool = batchmodels.CloudPool(
id=pool_id,
enable_inter_node_communication=True,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=image_ref_to_use, node_agent_sku_id=sku_to_use),
vm_size=vm_size,
target_dedicated=vm_count,
start_task=batchmodels.StartTask(
command_line=_STARTTASK_RESOURCE_FILE,
run_elevated=True,
wait_for_success=True,
resource_files=[
batchmodels.ResourceFile(file_path=_STARTTASK_RESOURCE_FILE,
blob_source=sas_url)
]),
)
common.helpers.create_pool_if_not_exist(batch_client, pool)
# because we want all nodes to be available before any tasks are assigned
# to the pool, here we will wait for all compute nodes to reach idle
nodes = common.helpers.wait_for_all_nodes_state(
batch_client, pool,
frozenset((batchmodels.ComputeNodeState.starttaskfailed,
batchmodels.ComputeNodeState.unusable,
batchmodels.ComputeNodeState.idle)))
# ensure all node are idle
if any(node.state != batchmodels.ComputeNodeState.idle for node in nodes):
raise RuntimeError('node(s) of pool {} not in idle state'.format(
pool.id))
return nodes
def create_pool(batch_service_client, pool_id, users):
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str pool_id: An ID for the new pool.
:param str publisher: Marketplace image publisher
:param str offer: Marketplace image offer
:param str sku: Marketplace image sky
"""
print('Creating pool [{}]...'.format(pool_id))
# Create a new pool of Linux compute nodes using an Azure Virtual Machines
# Marketplace image. For more information about creating pools of Linux
# nodes, see:
# https://azure.microsoft.com/documentation/articles/batch-linux-nodes/
# The start task installs ffmpeg on each node from an available repository, using
# an administrator user identity.
new_pool = batch.models.PoolAddParameter(
id=pool_id,
user_accounts=users,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=batchmodels.ImageReference(publisher="Canonical",
offer="UbuntuServer",
sku="18.04-LTS",
version="latest"),
node_agent_sku_id="batch.node.ubuntu 18.04"),
vm_size=_POOL_VM_SIZE,
target_dedicated_nodes=_DEDICATED_POOL_NODE_COUNT,
target_low_priority_nodes=_LOW_PRIORITY_POOL_NODE_COUNT,
start_task=batchmodels.StartTask(
#command_line="/bin/bash -c \"git clone https://github.com/uiuc-arc/probfuzz.git; \
# cd probfuzz/; ./install_java.sh; ./install.sh\"",
# command_line="/bin/bash -c \"apt-get update\"",
command_line="/bin/bash -c \"sudo apt-get -y update; \
sudo apt-get install git; \
sudo apt-get install -y python2.7; \
sudo apt-get install -y python-pip; \
sudo apt-get install -y bc; \
sudo apt-get install -y r-base; \
sudo pip2 --no-cache-dir install pandas; \
sudo pip2 install rpy2==2.8.6;\
sudo pip2 install argparse;\
sudo pip2 install numpy;\
sudo pip2 install scipy;\
\"",
wait_for_success=True,
user_identity=batchmodels.UserIdentity(user_name="admin"),
#user_identity=batchmodels.UserIdentity(
# auto_user=batchmodels.AutoUserSpecification(
# scope=batchmodels.AutoUserScope.pool,
# elevation_level=batchmodels.ElevationLevel.admin)),
),
max_tasks_per_node=_MAX_TASKS_PER_NODE)
batch_service_client.pool.add(new_pool)
def create_pool(batch_service_client, pool_id):
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str pool_id: An ID for the new pool.
:param str publisher: Marketplace image publisher
:param str offer: Marketplace image offer
:param str sku: Marketplace image sky
"""
print('Creating pool [{}]...'.format(pool_id))
# Create a new pool of Linux compute nodes using an Azure Virtual Machines
# Marketplace image. For more information about creating pools of Linux
# nodes, see:
# https://azure.microsoft.com/documentation/articles/batch-linux-nodes/
# The start task installs ffmpeg on each node from an available repository, using
# an administrator user identity.
#
task_commands = [
"apt-get update",
"apt-get -y install python3-pip",
"apt -y install htop",
"apt -y install iftop",
"pip3 install azure-storage-blob",
"pip3 install pyspark",
"pip3 install pandas",
"apt -y install openjdk-8-jre-headless"
]
new_pool = batch.models.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=batchmodels.ImageReference(
publisher="Canonical",
offer="UbuntuServer",
sku="18.04-LTS",
version="latest"
),
node_agent_sku_id="batch.node.ubuntu 18.04"),
vm_size=POOL_VM_SIZE,
target_dedicated_nodes=DEDICATED_POOL_NODE_COUNT,
target_low_priority_nodes=LOW_PRIORITY_POOL_NODE_COUNT,
start_task=batchmodels.StartTask(
command_line=wrap_commands_in_shell('linux',task_commands),
wait_for_success=True,
user_identity=batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)),
)
)
batch_service_client.pool.add(new_pool)
def generate_cluster_start_task(
core_base_operations,
zip_resource_file: batch_models.ResourceFile,
cluster_id: str,
gpu_enabled: bool,
docker_repo: str = None,
docker_run_options: str = None,
file_shares: List[models.FileShare] = None,
mixed_mode: bool = False,
worker_on_master: bool = True,
):
"""
This will return the start task object for the pool to be created.
:param cluster_id str: Id of the cluster(Used for uploading the resource files)
:param zip_resource_file: Resource file object pointing to the zip file containing scripts to run on the node
"""
resource_files = [zip_resource_file]
spark_web_ui_port = constants.DOCKER_SPARK_WEB_UI_PORT
spark_worker_ui_port = constants.DOCKER_SPARK_WORKER_UI_PORT
spark_job_ui_port = constants.DOCKER_SPARK_JOB_UI_PORT
spark_container_name = constants.DOCKER_SPARK_CONTAINER_NAME
spark_submit_logs_file = constants.SPARK_SUBMIT_LOGS_FILE
# TODO use certificate
environment_settings = (
__get_secrets_env(core_base_operations) + [
batch_models.EnvironmentSetting(name="SPARK_WEB_UI_PORT",
value=spark_web_ui_port),
batch_models.EnvironmentSetting(name="SPARK_WORKER_UI_PORT",
value=spark_worker_ui_port),
batch_models.EnvironmentSetting(name="SPARK_JOB_UI_PORT",
value=spark_job_ui_port),
batch_models.EnvironmentSetting(name="SPARK_CONTAINER_NAME",
value=spark_container_name),
batch_models.EnvironmentSetting(name="SPARK_SUBMIT_LOGS_FILE",
value=spark_submit_logs_file),
batch_models.EnvironmentSetting(
name="AZTK_GPU_ENABLED", value=helpers.bool_env(gpu_enabled)),
] + __get_docker_credentials(core_base_operations) +
_get_aztk_environment(cluster_id, worker_on_master, mixed_mode))
# start task command
command = __cluster_install_cmd(zip_resource_file, gpu_enabled,
docker_repo, docker_run_options,
file_shares)
return batch_models.StartTask(
command_line=helpers.wrap_commands_in_shell(command),
resource_files=resource_files,
environment_settings=environment_settings,
user_identity=POOL_ADMIN_USER_IDENTITY,
wait_for_success=True,
max_task_retry_count=2,
)
def create_pool(self, pool_id):
"""
Creates a pool of compute nodes with the specified OS settings.
:param str pool_id: An ID for the new pool.
:param dict config: Configuration details.
"""
if pool_id in self.active_pools or self.pool_exists(pool_id):
return
self.logger.info("creating pool {}".format(pool_id))
pool_config = self.config['pools'][pool_id]
sku_to_use, image_ref_to_use = self.__get_vm_image_and_node_agent_sku(
pool_config)
start_vm_commands = None
if pool_config.get('create_vm_commands', None):
start_vm_commands = self.__create_commands(
pool_config['create_vm_commands'])
user = batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)
vm_configuration = batchmodels.VirtualMachineConfiguration(
image_reference=image_ref_to_use,
node_agent_sku_id=sku_to_use,
)
vm_start_task = batchmodels.StartTask(
command_line=self.__wrap_commands_in_shell('linux',
start_vm_commands),
user_identity=batchmodels.UserIdentity(auto_user=user),
wait_for_success=True)
new_pool = batchmodels.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=vm_configuration,
vm_size=pool_config['pool_vm_size'],
enable_auto_scale=True,
auto_scale_formula=pool_config['auto_scale_formula'],
auto_scale_evaluation_interval=datetime.timedelta(minutes=5),
start_task=vm_start_task,
max_tasks_per_node=pool_config['max_tasks_per_node'],
)
try:
self.batch_client.pool.add(new_pool)
except batchmodels.BatchErrorException as err:
self.__print_batch_exception(err)
raise
self.active_pools.add(pool_id)
def generate_cluster_start_task(
spark_client,
zip_resource_file: batch_models.ResourceFile,
gpu_enabled: bool,
docker_repo: str = None,
file_shares: List[aztk_models.FileShare] = None):
"""
This will return the start task object for the pool to be created.
:param cluster_id str: Id of the cluster(Used for uploading the resource files)
:param zip_resource_file: Resource file object pointing to the zip file containing scripts to run on the node
"""
resource_files = [zip_resource_file]
spark_web_ui_port = constants.DOCKER_SPARK_WEB_UI_PORT
spark_worker_ui_port = constants.DOCKER_SPARK_WORKER_UI_PORT
spark_jupyter_port = constants.DOCKER_SPARK_JUPYTER_PORT
spark_job_ui_port = constants.DOCKER_SPARK_JOB_UI_PORT
spark_rstudio_server_port = constants.DOCKER_SPARK_RSTUDIO_SERVER_PORT
# TODO use certificate
environment_settings = [
batch_models.EnvironmentSetting(
name="BATCH_ACCOUNT_KEY", value=spark_client.batch_config.account_key),
batch_models.EnvironmentSetting(
name="BATCH_ACCOUNT_URL", value=spark_client.batch_config.account_url),
batch_models.EnvironmentSetting(
name="STORAGE_ACCOUNT_NAME", value=spark_client.blob_config.account_name),
batch_models.EnvironmentSetting(
name="STORAGE_ACCOUNT_KEY", value=spark_client.blob_config.account_key),
batch_models.EnvironmentSetting(
name="STORAGE_ACCOUNT_SUFFIX", value=spark_client.blob_config.account_suffix),
batch_models.EnvironmentSetting(
name="SPARK_WEB_UI_PORT", value=spark_web_ui_port),
batch_models.EnvironmentSetting(
name="SPARK_WORKER_UI_PORT", value=spark_worker_ui_port),
batch_models.EnvironmentSetting(
name="SPARK_JUPYTER_PORT", value=spark_jupyter_port),
batch_models.EnvironmentSetting(
name="SPARK_JOB_UI_PORT", value=spark_job_ui_port),
batch_models.EnvironmentSetting(
name="SPARK_RSTUDIO_SERVER_PORT", value=spark_rstudio_server_port),
] + __get_docker_credentials(spark_client)
# start task command
command = __cluster_install_cmd(zip_resource_file, gpu_enabled, docker_repo, file_shares)
return batch_models.StartTask(
command_line=helpers.wrap_commands_in_shell(command),
resource_files=resource_files,
environment_settings=environment_settings,
user_identity=POOL_ADMIN_USER_IDENTITY,
wait_for_success=True)
def create_pool_and_wait_for_node(batch_client, pool_id, vm_size, vm_count,
sha1_cert_tp):
"""Creates an Azure Batch pool with the specified id.
:param batch_client: The batch client to use.
:type batch_client: `batchserviceclient.BatchServiceClient`
:param str pool_id: The id of the pool to create.
:param str vm_size: vm size (sku)
:param int vm_count: number of vms to allocate
:param str sha1_cert_tp: sha1 cert thumbprint for cert ref
"""
# pick the latest supported 16.04 sku for UbuntuServer
sku_to_use, image_ref_to_use = \
common.helpers.select_latest_verified_vm_image_with_node_agent_sku(
batch_client, 'Canonical', 'UbuntuServer', '16.04')
# create start task commands
# 1. update repository
# 2. install blobxfer pre-requisites
# 3. pip install blobxfer python script
start_task_commands = [
'apt-get update',
'apt-get install -y build-essential libssl-dev libffi-dev ' +
'libpython-dev python-dev python-pip', 'pip install --upgrade blobxfer'
]
user = batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)
# create pool with start task and cert ref with visibility of task
pool = batchmodels.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=image_ref_to_use, node_agent_sku_id=sku_to_use),
vm_size=vm_size,
target_dedicated=vm_count,
start_task=batchmodels.StartTask(
command_line=common.helpers.wrap_commands_in_shell(
'linux', start_task_commands),
user_identity=batchmodels.UserIdentity(auto_user=user),
wait_for_success=True),
certificate_references=[
batchmodels.CertificateReference(
sha1_cert_tp,
'sha1',
visibility=[batchmodels.CertificateVisibility.task])
],
)
common.helpers.create_pool_if_not_exist(batch_client, pool)
def create_processing_pool(batch_service_client, start_task):
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:param str start_task: task start command.
:type batch_service_client: `azure.batch.BatchServiceClient`
"""
LOGGER.info(f'Creating pool [{PROCESSING_POOL_ID}]...')
image_ref_to_use = get_image_reference()
container_registry = \
batch_models.ContainerRegistry(
registry_server=REGISTRY_SERVER,
user_name=REGISTRY_ACCOUNT_USER,
password=REGISTRY_ACCOUNT_PASSWORD)
container_conf = batch_models.ContainerConfiguration(
container_image_names=[DOCKER_CONTAINER_URL],
container_registries=[container_registry])
new_pool = batch_models.PoolAddParameter(
id=PROCESSING_POOL_ID,
virtual_machine_configuration=
batch_models.VirtualMachineConfiguration(
image_reference=image_ref_to_use,
container_configuration=container_conf,
node_agent_sku_id=VM_AGENT_SKU),
vm_size=PROCESSING_POOL_VM_SIZE,
start_task=batch_models.StartTask(
command_line=start_task,
user_identity=batch_models.UserIdentity(
auto_user=batch_models.AutoUserSpecification(
scope='pool',
elevation_level='admin'))
),
enable_auto_scale=True,
auto_scale_evaluation_interval=datetime.timedelta(
minutes=PROCESSING_POOL_SCALE_INTERVAL_MINUTES),
auto_scale_formula=PROCESSING_POOL_SCALE_FORMULA)
try:
batch_service_client.pool.add(new_pool)
LOGGER.info("Processing Pool Created")
except batch_models.BatchErrorException as err:
if 'The specified pool already exists.' in err.error.message.value:
LOGGER.info("Pool already exists...")
else:
raise
def create_pool(batch_service_client, pool_id):
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str pool_id: An ID for the new pool.
:param str publisher: Marketplace image publisher
:param str offer: Marketplace image offer
:param str sku: Marketplace image sky
"""
print('Creating pool [{}]...'.format(pool_id))
# Create a new pool of Linux compute nodes using an Azure Virtual Machines
# Marketplace image. For more information about creating pools of Linux
# nodes, see:
# https://azure.microsoft.com/documentation/articles/batch-linux-nodes/
# The start task installs ffmpeg on each node from an available repository, using
# an administrator user identity.
new_pool = batch.models.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=batchmodels.ImageReference(publisher="Canonical",
offer="UbuntuServer",
sku="16.04-LTS",
version="latest"),
node_agent_sku_id="batch.node.ubuntu 16.04"),
vm_size=os.environ['POOL_VM_SIZE'],
target_dedicated_nodes=os.environ['DEDICATED_POOL_NODE_COUNT'],
target_low_priority_nodes=os.environ['LOW_PRIORITY_POOL_NODE_COUNT'],
start_task=batchmodels.StartTask(
command_line="/bin/bash -c \"apt-get update && \
apt-get install -y apt-transport-https ca-certificates curl software-properties-common &&\
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \
add-apt-repository 'deb [arch=amd64] https://download.docker.com/linux/ubuntu xenial stable' && \
apt-get update && \
apt-get install -y docker-ce\"",
wait_for_success=True,
user_identity=batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)),
))
batch_service_client.pool.add(new_pool)
def create_pool_and_wait_for_nodes(batch_client, block_blob_client, pool_id,
vm_size, vm_count):
sku_to_use, image_ref_to_use = common.helpers.select_latest_verified_vm_image_with_node_agent_sku(
batch_client, 'Canonical', 'UbuntuServer', '14.04')
block_blob_client.create_container(_CONTAINER_NAME, fail_on_exist=False)
# upload start task script
block_blob_client.create_container(_CONTAINER_NAME, fail_on_exist=False)
sas_url = common.helpers.upload_blob_and_create_sas(
block_blob_client, _CONTAINER_NAME, _STARTTASK_RESOURCE_FILE,
_STARTTASK_SHELL_SCRIPT_PATH,
datetime.datetime.utcnow() + datetime.timedelta(hours=1))
# create pool and execute starttask
pool = batchmodels.PoolAddParameter(
id=pool_id,
enable_inter_node_communication=True,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=image_ref_to_use, node_agent_sku_id=sku_to_use),
vm_size=vm_size,
target_dedicated=vm_count,
start_task=batchmodels.StartTask(
command_line=_STARTTASK_RESOURCE_FILE,
run_elevated=True,
wait_for_success=True,
resource_files=[
batchmodels.ResourceFile(file_path=_STARTTASK_RESOURCE_FILE,
blob_source=sas_url)
]),
)
common.helpers.create_pool_if_not_exist(batch_client, pool)
# because we want all nodes to be available before any tasks are assigned
# to the pool, here we will wait for all compute nodes to reach idle
nodes = common.helpers.wait_for_all_nodes_state(
batch_client, pool,
frozenset((batchmodels.ComputeNodeState.starttaskfailed,
batchmodels.ComputeNodeState.unusable,
batchmodels.ComputeNodeState.idle)))
# ensure all node are idle
if any(node.state != batchmodels.ComputeNodeState.idle for node in nodes):
raise RuntimeError('node(s) of pool {} not in idle state'.format(
pool.id))
return nodes
def create_pool(batch_service_client, pool_id, application_files):
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str pool_id: An ID for the new pool.
:param str publisher: Marketplace image publisher
:param str offer: Marketplace image offer
:param str sku: Marketplace image sky
"""
print('Creating pool [{}]...'.format(pool_id))
# Create a new pool of Linux compute nodes using an Azure Virtual Machines
# Marketplace image. For more information about creating pools of Linux
# nodes, see:
# https://azure.microsoft.com/documentation/articles/batch-linux-nodes/
# The start task installs ffmpeg on each node from an available repository, using
# an administrator user identity.
new_pool = batch.models.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=batchmodels.ImageReference(
publisher="MicrosoftWindowsServer",
offer="WindowsServer",
sku="2016-Datacenter",
version="latest"
),
node_agent_sku_id="batch.node.windows amd64"),
vm_size=config._POOL_VM_SIZE,
target_dedicated_nodes=config._DEDICATED_POOL_NODE_COUNT,
target_low_priority_nodes=config._LOW_PRIORITY_POOL_NODE_COUNT,
start_task=batchmodels.StartTask(
command_line="cmd /c starttask.cmd",
resource_files=application_files,
wait_for_success=True,
user_identity=batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)),
)
)
batch_service_client.pool.add(new_pool)
def create_pool(batch_service_client, pool_id):
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str pool_id: An ID for the new pool.
:param str publisher: Marketplace image publisher
:param str offer: Marketplace image offer
:param str sku: Marketplace image sky
"""
print('Creating pool [{}]...'.format(pool_id))
# Create a new pool of Linux compute nodes using an Azure Virtual Machines
# Marketplace image. For more information about creating pools of Linux
# nodes, see:
# https://azure.microsoft.com/documentation/articles/batch-linux-nodes/
# The start task installs ffmpeg on each node from an available repository, using
# an administrator user identity.
new_pool = batch.models.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=batchmodels.ImageReference(publisher="Canonical",
offer="UbuntuServer",
sku="18.04-LTS",
version="latest"),
node_agent_sku_id="batch.node.ubuntu 18.04"),
vm_size=config._POOL_VM_SIZE,
target_dedicated_nodes=config._DEDICATED_POOL_NODE_COUNT,
target_low_priority_nodes=config._LOW_PRIORITY_POOL_NODE_COUNT,
start_task=batchmodels.StartTask(
command_line=
"/bin/bash -c \"apt-get update && apt-get install wget && wget http://cab.spbu.ru/files/release3.14.0/SPAdes-3.14.0-Linux.tar.gz && tar -xf SPAdes-3.14.0-Linux.tar.gz\"",
wait_for_success=True,
user_identity=batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)),
))
batch_service_client.pool.add(new_pool)
def create_pool(batch_service_client: batch.BatchServiceClient,
pool_id: str,
publisher: str = "Canonical",
offer: str = "UbuntuServer",
sku: str = "18.04-LTS") -> None:
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:param pool_id: An ID for the new pool.
:param publisher: Marketplace image publisher
:param offer: Marketplace image offer
:param sku: Marketplace image sky
"""
print('Creating pool [{}]...'.format(pool_id))
# Create a new pool of Linux compute nodes using an Azure Virtual Machines
# Marketplace image. For more information about creating pools of Linux
# nodes, see:
# https://azure.microsoft.com/documentation/articles/batch-linux-nodes/
new_pool = batch.models.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=batchmodels.ImageReference(publisher=publisher,
offer=offer,
sku=sku,
version="latest"),
node_agent_sku_id="batch.node.ubuntu 18.04"),
vm_size=config._POOL_VM_SIZE,
target_dedicated_nodes=config._DEDICATED_POOL_NODE_COUNT,
target_low_priority_nodes=config._LOW_PRIORITY_POOL_NODE_COUNT,
start_task=batchmodels.StartTask(
command_line=
"/bin/bash -c \"apt-get update && apt-get -y install python3.7 python3-pip\"",
wait_for_success=True,
user_identity=batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)),
))
batch_service_client.pool.add(new_pool)
def createBatchPool(batch_client, pool_id):
start_cmd = "/bin/bash -c \"apt-get install -y python3-pip python3-venv\""
admin = batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(elevation_level='admin'))
new_pool = batch.models.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=batchmodels.ImageReference(publisher="Canonical",
offer="UbuntuServer",
sku="18.04-LTS",
version="latest"),
node_agent_sku_id="batch.node.ubuntu 18.04"),
vm_size=
'STANDARD_A2m_v2', # VM Type/Size # STANDARD_A2m_v2 16 GB # Standard_E4_v3 32 GB
target_dedicated_nodes=1, # pool node count
start_task=batchmodels.StartTask(command_line=start_cmd,
user_identity=admin))
batch_client.pool.add(new_pool)
def create_pool(batch_client, container_conf, container_settings, image_ref_to_use, pool_id, sku_to_use,
vm_count, vm_size):
start_task_settings = container_settings
start_task_settings.working_directory = ContainerWorkingDirectory.container_image_default
pool = batchmodels.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=image_ref_to_use,
container_configuration=container_conf,
node_agent_sku_id=sku_to_use),
vm_size=vm_size,
max_tasks_per_node=1,
target_dedicated_nodes=vm_count,
start_task=batchmodels.StartTask(
command_line="",
wait_for_success=True,
container_settings=start_task_settings),
)
azure_helpers.create_pool_if_not_exist(batch_client, pool)
def create_pool(batch_client, block_blob_client, pool_id, vm_size, vm_count):
"""Creates an Azure Batch pool with the specified id.
:param batch_client: The batch client to use.
:type batch_client: `batchserviceclient.BatchServiceClient`
:param block_blob_client: The storage block blob client to use.
:type block_blob_client: `azure.storage.blob.BlockBlobService`
:param str pool_id: The id of the pool to create.
:param str vm_size: vm size (sku)
:param int vm_count: number of vms to allocate
"""
# pick the latest supported 14.04 sku for UbuntuServer
sku_to_use, image_ref_to_use = \
common.helpers.select_latest_verified_vm_image_with_node_agent_sku(
batch_client, 'Canonical', 'UbuntuServer', '14.04')
block_blob_client.create_container(
_CONTAINER_NAME,
fail_on_exist=False)
sas_url = common.helpers.upload_blob_and_create_sas(
block_blob_client,
_CONTAINER_NAME,
_SIMPLE_TASK_NAME,
_SIMPLE_TASK_PATH,
datetime.datetime.utcnow() + datetime.timedelta(hours=1))
pool = batchmodels.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=image_ref_to_use,
node_agent_sku_id=sku_to_use),
vm_size=vm_size,
target_dedicated=vm_count,
start_task=batchmodels.StartTask(
command_line="python " + _SIMPLE_TASK_NAME,
resource_files=[batchmodels.ResourceFile(
file_path=_SIMPLE_TASK_NAME,
blob_source=sas_url)]))
common.helpers.create_pool_if_not_exist(batch_client, pool)
def create_commit_pool(batch_service_client):
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
"""
LOGGER.info(f'Creating pool [{PROCESSING_POOL_ID}]...')
image_ref_to_use = batch_models.ImageReference(
publisher='canonical',
offer='ubuntuserver',
sku='18.04-lts',
version='latest')
new_pool = batch_models.PoolAddParameter(
id=COMMIT_POOL_ID,
virtual_machine_configuration=
batch_models.VirtualMachineConfiguration(
image_reference=image_ref_to_use,
node_agent_sku_id="batch.node.ubuntu 18.04"),
vm_size=COMMIT_POOL_VM_SIZE,
start_task=batch_models.StartTask(
command_line=COMMIT_POOL_START_TASK,
user_identity=batch_models.UserIdentity(
auto_user=batch_models.AutoUserSpecification(
scope='pool',
elevation_level='admin'))
),
enable_auto_scale=True,
auto_scale_evaluation_interval=datetime.timedelta(
minutes=COMMIT_POOL_SCALE_INTERVAL_MINUTES),
auto_scale_formula=COMMIT_POOL_SCALE_FORMULA)
try:
batch_service_client.pool.add(new_pool)
LOGGER.info("Commit Pool Created")
except batch_models.BatchErrorException as err:
if 'The specified pool already exists.' in err.error.message.value:
LOGGER.info("Pool already exists...")
else:
raise
def _create_start_task(command, resource_files):
"""The start task is run each node as it joins the pool, and when it's rebooted or re-imaged to prep for
any jobs later. The script is run "shared" directory that all tasks that run on the node have access to.
Args:
command: a string containing all the commands to run as the start task
resource_files: list of file references (of type azure.batch.models.ResourceFile)
from Azure Storage to download to each node
Returns:
azure.batch.models.StartTask object
"""
user = batch_models.AutoUserSpecification(
scope=batch_models.AutoUserScope.pool,
elevation_level=batch_models.ElevationLevel.admin)
return batch_models.StartTask(
command_line=command,
user_identity=batch_models.UserIdentity(auto_user=user),
wait_for_success=True,
resource_files=resource_files)
def create_pool(batch_client, block_blob_client, pool_id, vm_size, vm_count):
"""Creates an Azure Batch pool with the specified id.
:param batch_client: The batch client to use.
:type batch_client: `batchserviceclient.BatchServiceClient`
:param block_blob_client: The storage block blob client to use.
:type block_blob_client: `azure.storage.blob.BlockBlobService`
:param str pool_id: The id of the pool to create.
:param str vm_size: vm size (sku)
:param int vm_count: number of vms to allocate
"""
# pick the latest supported 16.04 sku for UbuntuServer
sku_to_use, image_ref_to_use = \
common.helpers.select_latest_verified_vm_image_with_node_agent_sku(
batch_client, 'Canonical', 'UbuntuServer', '18.04')
block_blob_client.create_container(_CONTAINER_NAME, fail_on_exist=False)
sas_url = common.helpers.upload_blob_and_create_sas(
block_blob_client, _CONTAINER_NAME, _SIMPLE_TASK_NAME,
_SIMPLE_TASK_PATH, _EXPIRY_TIME)
start_tasks = []
pool = batchmodels.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=image_ref_to_use, node_agent_sku_id=sku_to_use),
vm_size=vm_size,
target_dedicated_nodes=vm_count,
start_task=batchmodels.StartTask(
command_line=
'/bin/bash -c \"sudo apt-get -y update && export DEBIAN_FRONTEND=noninteractive && sudo apt-get install -y python3-pip && sudo pip3 install numpy statsmodels pmdarima\"',
wait_for_success=True,
user_identity=batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)),
))
common.helpers.create_pool_if_not_exist(batch_client, pool)
def create_pool(batch_service_client, pool_id):
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str pool_id: An ID for the new pool.
:param str publisher: Marketplace image publisher
:param str offer: Marketplace image offer
:param str sku: Marketplace image sky
"""
print('Creating pool [{}]...'.format(pool_id))
# Create a new pool of Linux compute nodes using an Azure Virtual Machines
# Marketplace image. For more information about creating pools of Linux
# nodes, see:
# https://azure.microsoft.com/documentation/articles/batch-linux-nodes/
try:
new_pool = batch.models.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.
VirtualMachineConfiguration(
image_reference=batchmodels.ImageReference(
publisher="Canonical",
offer="UbuntuServer",
sku="16.04-LTS",
version="latest"),
node_agent_sku_id="batch.node.ubuntu 16.04"),
vm_size=_POOL_VM_SIZE,
target_dedicated_nodes=_POOL_NODE_COUNT,
start_task=batchmodels.StartTask(
command_line=
"/bin/bash -c \"sudo apt-get update && sudo apt-get -y install python3-pip build-essential libssl-dev libffi-dev python3-dev && sudo pip3 install azure;\"",
user_identity=user,
wait_for_success=True))
batch_service_client.pool.add(new_pool)
except batchmodels.batch_error.BatchErrorException as err:
print_batch_exception(err)
def create_pool(batch_service_client, pool_id):
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str pool_id: An ID for the new pool.
:param str publisher: Marketplace image publisher
:param str offer: Marketplace image offer
:param str sku: Marketplace image sky
"""
print('Creating pool [{}]...'.format(pool_id))
# The start task downloads and installs ilastik on each node and cd
# into the ilastik directory (which resides in the nodes shared directory)
command_line = "/bin/bash -c \"wget {} -P $AZ_BATCH_NODE_SHARED_DIR && cd $AZ_BATCH_NODE_SHARED_DIR && tar xjf ilastik*.tar.bz2\"".format(
_ILASTIK_DOWNLOAD_URL)
new_pool = batch.models.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=batchmodels.ImageReference(publisher="Canonical",
offer="UbuntuServer",
sku="16.04.0-LTS",
version="latest"),
node_agent_sku_id="batch.node.ubuntu 16.04"),
vm_size=_POOL_VM_SIZE,
target_dedicated_nodes=_DEDICATED_POOL_NODE_COUNT,
start_task=batchmodels.StartTask(
command_line=command_line,
wait_for_success=True,
user_identity=batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)),
))
batch_service_client.pool.add(new_pool)
def create_pool(batch_client, block_blob_client, pool_id, vm_size, vm_count):
block_blob_client.create_container(CONTAINER_NAME, fail_on_exist=False)
sku_to_use, image_ref_to_use = select_latest_vm_image_with_node_agent_sku(
batch_client, 'Canonical', 'UbuntuServer', '18.04')
sas_url = upload_blob_and_create_sas(
block_blob_client, CONTAINER_NAME, TASK_NAME, TASK_PATH,
datetime.datetime.utcnow() + datetime.timedelta(hours=1))
pool = batchmodels.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=image_ref_to_use, node_agent_sku_id=sku_to_use),
vm_size=vm_size,
target_dedicated_nodes=vm_count,
start_task=batchmodels.StartTask(command_line="python3 " + TASK_NAME,
resource_files=[
batchmodels.ResourceFile(
file_path=TASK_NAME,
blob_source=sas_url)
]))
create_pool_if_not_exist(batch_client, pool)
def test_batch_update_pools(self, **kwargs):
client = self.create_sharedkey_client(**kwargs)
# Test Create Paas Pool
test_paas_pool = models.PoolAddParameter(
id=self.get_resource_name('batch_paas_'),
vm_size='small',
cloud_service_configuration=models.CloudServiceConfiguration(
os_family='5'
),
start_task=models.StartTask(
command_line="cmd.exe /c \"echo hello world\"",
resource_files=[models.ResourceFile('https://blobsource.com', 'filename.txt')],
environment_settings=[models.EnvironmentSetting('ENV_VAR', 'env_value')],
user_identity=models.UserIdentity(
auto_user=models.AutoUserSpecification(
elevation_level=models.ElevationLevel.admin
)
)
)
)
response = client.pool.add(test_paas_pool)
self.assertIsNone(response)
# Test Upgrade Pool OS
self.assertBatchError(
"PoolVersionEqualsUpgradeVersion",
client.pool.upgrade_os,
test_paas_pool.id,
"*"
)
# Test Update Pool Parameters
params = models.PoolUpdatePropertiesParameter([], [], [models.MetadataItem('foo', 'bar')])
response = client.pool.update_properties(test_paas_pool.id, params)
self.assertIsNone(response)
# Test Patch Pool Parameters
params = models.PoolPatchParameter(metadata=[models.MetadataItem('foo2', 'bar2')])
response = client.pool.patch(test_paas_pool.id, params)
self.assertIsNone(response)
# Test Pool Exists
response = client.pool.exists(test_paas_pool.id)
self.assertTrue(response)
# Test Get Pool
pool = client.pool.get(test_paas_pool.id)
self.assertIsInstance(pool, models.CloudPool)
self.assertEqual(pool.id, test_paas_pool.id)
self.assertEqual(pool.state, models.PoolState.active)
self.assertEqual(pool.allocation_state, models.AllocationState.steady)
self.assertEqual(pool.cloud_service_configuration.os_family, '5')
self.assertEqual(pool.vm_size, 'small')
self.assertIsNone(pool.start_task)
self.assertEqual(pool.metadata[0].name, 'foo2')
self.assertEqual(pool.metadata[0].value, 'bar2')
# Test Get Pool with OData Clauses
options = models.PoolGetOptions(select='id,state', expand='stats')
pool = client.pool.get(test_paas_pool.id, options)
self.assertIsInstance(pool, models.CloudPool)
self.assertEqual(pool.id, test_paas_pool.id)
self.assertEqual(pool.state, models.PoolState.active)
self.assertIsNone(pool.allocation_state)
self.assertIsNone(pool.vm_size)
# Test Delete Pool
response = client.pool.delete(test_paas_pool.id)
self.assertIsNone(response)
def create_or_update_pool(batch_service_client, pool_id, num_tasks):
"""
Creates or updates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str pool_id: An ID for the new pool.
:param num_tasks: the number of tasks that will be added to the pool later (to compute amount of nodes).
"""
logging.info('Creating pool [{}]...'.format(pool_id))
maxNumberofVMs = 25
pool_size = min(maxNumberofVMs, num_tasks)
if batch_service_client.pool.exists(pool_id) == False:
new_pool = batch.models.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.
VirtualMachineConfiguration(
image_reference=batchmodels.ImageReference(
publisher="Canonical",
offer="UbuntuServer",
sku="18.04-LTS",
version="latest"),
node_agent_sku_id="batch.node.ubuntu 18.04"),
vm_size=os.environ["_POOL_VM_SIZE"],
enable_auto_scale=True,
auto_scale_evaluation_interval=timedelta(days=0,
hours=0,
minutes=5),
auto_scale_formula='''// Sets the initial pool size:
initialPoolSize = {poolsize};
$TargetLowPriorityNodes = initialPoolSize;
// "Mon, 06 Oct 2014 10:20:00 GMT" represents the datetime that this autoscale formula starts to evaluate. This is an arbitrary value here.
lifespan = time() - time("Mon, 06 Oct 2014 10:20:00 GMT");
// Representing 15 minutes
span = TimeInterval_Minute * 15;
// Representing 10 minutes
startup = TimeInterval_Minute * 10;
ratio = 50;
// After 10 minutes, obtains the max value of the number of running and active tasks within the past 15 minutes.
// If both values are 0 (indicating that no tasks were running or active in the last 15 minutes), the pool size is set to 0.
// If either value is greater than zero, no change is made.
$TargetLowPriorityNodes = (lifespan > startup ? (max($RunningTasks.GetSample(span, ratio),
$ActiveTasks.GetSample(span, ratio)) == 0 ? 0 : $TargetLowPriorityNodes) : initialPoolSize );
$NodeDeallocationOption = taskcompletion;'''.format(
poolsize=pool_size, now=datetime.datetime.now()),
start_task=batchmodels.StartTask(
command_line=
"/bin/bash -c \"add-apt-repository ppa:deadsnakes/ppa && apt-get update && apt-get install -y ffmpeg python3.7 python3.7-venv python3.7-dev && apt-get install -y python3-pip && apt-get install -f && python3.7 -m pip install --upgrade pip setuptools wheel && pip3 install python-dateutil && pip3 install psutil && pip3 install requests && pip3 install tzlocal && pip3 install tesla-dashcam\"",
wait_for_success=True,
user_identity=batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)),
))
batch_service_client.pool.add(new_pool)
logging.info('Pool created.')
else:
logging.info('Pool already exists! Resizing..')
pool_resize_parameter = batch.models.PoolResizeParameter(
target_dedicated_nodes=0, target_low_priority_nodes=pool_size)
batch_service_client.pool.resize(pool_id, pool_resize_parameter)
def create_job_schedule(batch_client, job_schedule_id, vm_size, vm_count,
block_blob_client):
"""Creates an Azure Batch pool and job schedule with the specified ids.
:param batch_client: The batch client to use.
:type batch_client: `batchserviceclient.BatchServiceClient`
:param str job_schedule_id: The id of the job schedule to create
:param str vm_size: vm size (sku)
:param int vm_count: number of vms to allocate
:param block_blob_client: The storage block blob client to use.
:type block_blob_client: `azure.storage.blob.BlockBlobService`
"""
cloud_service_config = batchmodels.CloudServiceConfiguration(os_family='6')
user_id = batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
elevation_level=_USER_ELEVATION_LEVEL))
python_download = batchmodels.ResourceFile(http_url=_PYTHON_DOWNLOAD,
file_path='python373.exe')
pool_info = batchmodels.PoolInformation(
auto_pool_specification=batchmodels.AutoPoolSpecification(
auto_pool_id_prefix="JobScheduler",
pool=batchmodels.PoolSpecification(
vm_size=vm_size,
target_dedicated_nodes=vm_count,
cloud_service_configuration=cloud_service_config,
start_task=batchmodels.StartTask(
command_line=common.helpers.wrap_commands_in_shell(
'windows', ['{}'.format(_PYTHON_INSTALL)]),
resource_files=[python_download],
wait_for_success=True,
user_identity=user_id)),
keep_alive=False,
pool_lifetime_option=batchmodels.PoolLifetimeOption.job))
sas_url = common.helpers.upload_blob_and_create_sas(
block_blob_client, _CONTAINER_NAME, _SIMPLE_TASK_NAME,
_SIMPLE_TASK_PATH,
datetime.datetime.utcnow() + datetime.timedelta(minutes=30))
job_spec = batchmodels.JobSpecification(
pool_info=pool_info,
# Terminate job once all tasks under it are complete to allow for a new
# job to be created under the schedule
on_all_tasks_complete=batchmodels.OnAllTasksComplete.terminate_job,
job_manager_task=batchmodels.JobManagerTask(
id="JobManagerTask",
command_line=common.helpers.wrap_commands_in_shell(
'windows', ['python {}'.format(_SIMPLE_TASK_NAME)]),
resource_files=[
batchmodels.ResourceFile(file_path=_SIMPLE_TASK_NAME,
http_url=sas_url)
]))
do_not_run_after = datetime.datetime.utcnow() \
+ datetime.timedelta(minutes=30)
schedule = batchmodels.Schedule(
do_not_run_after=do_not_run_after,
recurrence_interval=datetime.timedelta(minutes=10))
scheduled_job = batchmodels.JobScheduleAddParameter(
id=job_schedule_id, schedule=schedule, job_specification=job_spec)
batch_client.job_schedule.add(cloud_job_schedule=scheduled_job)
def check_or_create_pool(self, id=None):
if id is None:
id = self.config.get('POOL', 'id')
self.pool_id = id
if self.client.pool.exists(id):
found_job = False
# Update the Job ID here
for job in self.client.job.list():
if job.pool_info.pool_id == self.pool_id:
self.job_id = job.id
found_job = True
break
if not found_job:
self.start_mc_server_job_pool(
) # Restart Jobs for this pool - this is necessary!
return self.client.pool.get(id)
api_port = self.config.get('POOL', 'api_port')
min_count = self.config.get('POOL', 'mincount')
image_reference = batchmodels.ImageReference(
virtual_machine_image_id=
"/subscriptions/889566d5-6e5d-4d31-a82d-b60603b3e50b/resourceGroups/polycraft-game/providers/Microsoft.Compute/galleries/polycraftImgGallery/images/polycraftBestGameServerV1/versions/1.0.0"
)
vmc = batchmodels.VirtualMachineConfiguration(
image_reference=image_reference,
node_agent_sku_id="batch.node.ubuntu 18.04")
users = [
batchmodels.UserAccount(
name='azureuser',
password='******',
elevation_level=batchmodels.ElevationLevel.admin),
# batchmodels.UserAccount(
# name='pool-nonadmin',
# password='******',
# elevation_level=batchmodels.ElevationLevel.non_admin)
]
# Thank you Ask Ubuntu https://askubuntu.com/a/373478
wait_for_locks = 'while sudo fuser /var/lib/dpkg/lock /var/lib/apt/lists/lock /var/cache/apt/archives/lock /var/lib/dpkg/lock-frontend >/dev/null 2>&1; do echo "Waiting for release of apt locks"; sleep 2; done; '
# NOTE: Always use DOUBLE QUOTES within commands as azure prepends the entire string with a single quote.
start_task = batchmodels.StartTask(
command_line=helpers.wrap_commands_in_shell(
'linux',
[
'whoami',
'printenv',
'usermod -aG sudo azureuser',
'sudo systemctl disable --now apt-daily.timer',
'sudo systemctl disable --now apt-daily-upgrade.timer',
'sudo systemctl daemon-reload',
'cd /home/polycraft',
'chmod -R 777 *',
'rm /home/polycraft/oxygen/mods/*.jar',
'cd /home/polycraft/oxygen/',
'echo "[DEBUG] removing helium..."',
'ls -l',
f'sudo rm -rf /home/polycraft/oxygen/{self.config.get("SERVER","worldName")}',
'sudo rm -f *.zip',
'echo "[DEBUG] removed helium?"',
'ls -l',
# Stop the crontabs from running
'sudo rm /var/spool/cron/crontabs/*',
# Taken from: https://stackoverflow.com/questions/45269225/ansible-playbook-fails-to-lock-apt/51919678#51919678
'sudo systemd-run --property="After=apt-daily.service apt-daily-upgrade.service" --wait /bin/true',
'sudo apt-get -y purge unattended-upgrades',
'sudo apt-get -y update',
wait_for_locks +
'sudo apt-get install software-properties-common -y',
# 'while fuser /var/lib/dpkg/lock >/dev/null 2>&1; do sleep 1; done; sudo apt-add-repository universe',
wait_for_locks + 'sudo apt-add-repository universe',
# Mount the Polycraft Game FileShare
wait_for_locks +
'sudo apt-get install cifs-utils -y && sudo mkdir -p /mnt/PolycraftGame/',
f'mount -t cifs //polycraftbestbatch.file.core.windows.net/best-batch-round-1-test /mnt/PolycraftGame -o vers=3.0,username={self.credentials.get("Storage", "storageaccountname")},password={self.credentials.get("Storage", "storageaccountkey")},dir_mode=0777,file_mode=0777,serverino && ls /mnt/PolycraftGame',
# Copy the default world file to the right folder
f'cp /mnt/PolycraftGame/{self.config.get("SERVER","fileShareFolder")}/worlds/{self.config.get("SERVER","worldZipName")}.tar.gz /home/polycraft/oxygen/',
'cd /home/polycraft/oxygen/',
# 'sudo rm -r helium',
f'gzip -d /home/polycraft/oxygen/{self.config.get("SERVER","worldZipName")}.tar.gz',
'echo "[DEBUG] extracting the tar"',
'ls -l',
f'sudo tar -xf {self.config.get("SERVER","worldZipName")}.tar',
'echo "[DEBUG] extracted the tar"',
'ls -l',
# 'sudo mv helium-backup-0924 helium',
f'sudo mv helium {self.config.get("SERVER","worldName")}', # TODO Remove this once we finalize the server name?
f'chmod -R 777 {self.config.get("SERVER","worldName")}/', # NOTE: The folder inside here is called helium!
'echo "[DEBUG] Adjusted permissions for helium?"',
'ls -l',
]),
wait_for_success=True,
# user_accounts=users,
user_identity=batchmodels.UserIdentity(
# user_name='azureuser',
auto_user=batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)
# ),
),
)
net_config = batchmodels.NetworkConfiguration(
# subnet_id="/subscriptions/889566d5-6e5d-4d31-a82d-b60603b3e50b/resourceGroups/vnet-eastus-azurebatch/providers/Microsoft.Network/virtualNetworks/vnet-eastus-azurebatch/subnets/main-batch-subnet",
endpoint_configuration=batchmodels.
PoolEndpointConfiguration(inbound_nat_pools=[
batchmodels.InboundNATPool(
name='minecraftServer',
protocol='tcp',
backend_port=25565,
frontend_port_range_start=44000,
frontend_port_range_end=44099,
network_security_group_rules=[
batchmodels.NetworkSecurityGroupRule(
priority=199,
access='allow',
source_address_prefix='*'),
]),
batchmodels.InboundNATPool(
name='api_port',
protocol='tcp',
backend_port=int(api_port)
if api_port and api_port.isdecimal() else 9007,
frontend_port_range_start=44500,
frontend_port_range_end=44599,
network_security_group_rules=[
# batchmodels.NetworkSecurityGroupRule(
# priority=170,
# access='allow',
# source_address_prefix='192.168.1.0/24' # TODO: is this the right subnet?
# ),
batchmodels.NetworkSecurityGroupRule(
priority=198,
access='allow', # 'deny'
source_address_prefix=
'*' # TODO: only allow access to the right ports
)
]),
]))
pool = batchmodels.PoolAddParameter(
id=id,
vm_size=self.config.get('POOL', 'vm_size'),
target_dedicated_nodes=int(min_count)
if min_count and min_count.isdecimal() else 1,
virtual_machine_configuration=vmc,
start_task=start_task,
user_accounts=users,
network_configuration=net_config)
helpers.create_pool_if_not_exist(self.client, pool)
self.start_mc_server_job_pool(pool.target_dedicated_nodes)
def create_pool_and_wait_for_node(batch_client, pool_id, vm_size, vm_count,
sha1_cert_tp):
"""Creates an Azure Batch pool with the specified id.
:param batch_client: The batch client to use.
:type batch_client: `batchserviceclient.BatchServiceClient`
:param str pool_id: The id of the pool to create.
:param str vm_size: vm size (sku)
:param int vm_count: number of vms to allocate
:param str sha1_cert_tp: sha1 cert thumbprint for cert ref
"""
# pick the latest supported 16.04 sku for UbuntuServer
sku_to_use, image_ref_to_use = \
common.helpers.select_latest_verified_vm_image_with_node_agent_sku(
batch_client, 'Canonical', 'UbuntuServer', '16.04')
# create start task commands
# 1. update repository
# 2. install blobxfer pre-requisites
# 3. pip install blobxfer python script
start_task_commands = [
'apt-get update',
'apt-get install -y build-essential libssl-dev libffi-dev ' +
'libpython-dev python-dev python-pip', 'pip install --upgrade blobxfer'
]
user = batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)
# create pool with start task and cert ref with visibility of task
pool = batchmodels.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=image_ref_to_use, node_agent_sku_id=sku_to_use),
vm_size=vm_size,
target_dedicated_nodes=vm_count,
start_task=batchmodels.StartTask(
command_line=common.helpers.wrap_commands_in_shell(
'linux', start_task_commands),
user_identity=batchmodels.UserIdentity(auto_user=user),
wait_for_success=True),
certificate_references=[
batchmodels.CertificateReference(
thumbprint=sha1_cert_tp,
thumbprint_algorithm='sha1',
visibility=[batchmodels.CertificateVisibility.task])
],
)
common.helpers.create_pool_if_not_exist(batch_client, pool)
# because we want all nodes to be available before any tasks are assigned
# to the pool, here we will wait for all compute nodes to reach idle
nodes = common.helpers.wait_for_all_nodes_state(
batch_client, pool,
frozenset((batchmodels.ComputeNodeState.start_task_failed,
batchmodels.ComputeNodeState.unusable,
batchmodels.ComputeNodeState.idle)))
# ensure all node are idle
if any(node.state != batchmodels.ComputeNodeState.idle for node in nodes):
raise RuntimeError(
'node(s) of pool {} not in idle state'.format(pool_id))
def create_pool(self,
pool_id,
vm_size,
target_dedicated,
target_low_priority,
batch_image_spec,
starttask_cmd,
starttask_url,
starttask_script,
sp_cert_thumb,
app_licenses=None,
disable_remote_access=True,
app_pkgs=None,
subnet_id=None,
app_insights_app_key=None,
app_insights_instrumentation_key=None):
pool = batchmodels.PoolAddParameter(
id=pool_id,
display_name=pool_id,
vm_size=vm_size,
target_dedicated_nodes=target_dedicated,
target_low_priority_nodes=target_low_priority,
virtual_machine_configuration=batch_image_spec.
get_virtual_machine_configuration(),
application_package_references=app_pkgs,
certificate_references=[
batchmodels.CertificateReference(sp_cert_thumb, 'sha1')
])
if app_licenses:
pool.application_licenses = app_licenses
pool.start_task = batchmodels.StartTask(
command_line=starttask_cmd,
max_task_retry_count=3,
user_identity=batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)),
wait_for_success=True,
resource_files=[
batchmodels.ResourceFile(starttask_url, starttask_script)
])
if app_insights_app_key and app_insights_instrumentation_key:
pool.start_task.environment_settings = [
batchmodels.EnvironmentSetting('APP_INSIGHTS_APP_ID',
app_insights_app_key),
batchmodels.EnvironmentSetting(
'APP_INSIGHTS_INSTRUMENTATION_KEY',
app_insights_instrumentation_key)
]
if subnet_id:
pool.network_configuration = batchmodels.NetworkConfiguration(
subnet_id=subnet_id)
if disable_remote_access:
if pool.network_configuration is None:
pool.network_configuration = batchmodels.NetworkConfiguration()
endpoint_config = batchmodels.PoolEndpointConfiguration(
inbound_nat_pools=[
batchmodels.InboundNATPool(
'DisableRDP',
batchmodels.InboundEndpointProtocol.tcp,
3389,
60000,
60099,
network_security_group_rules=[
batchmodels.NetworkSecurityGroupRule(
150, batchmodels.
NetworkSecurityGroupRuleAccess.deny, '*')
]),
batchmodels.InboundNATPool(
'DisableSSH',
batchmodels.InboundEndpointProtocol.tcp,
22,
61000,
61099,
network_security_group_rules=[
batchmodels.NetworkSecurityGroupRule(
151, batchmodels.
NetworkSecurityGroupRuleAccess.deny, '*')
])
])
pool.network_configuration.endpoint_configuration = endpoint_config
try:
client = self._get_batch_client()
client.pool.add(pool)
except batchmodels.BatchErrorException as be:
if be.error:
print('Error creating pool, code={}, message={}'.format(
be.error.code, be.error.message))
if be.error.values:
for e in be.error.values:
print('Key={}, Value={}'.format(e.key, e.value))
raise