def pool_create():
image_reference = batchmodel.ImageReference(
publisher=config_azure['batch_pool_image_publisher'],
offer=config_azure['batch_pool_image_offer'],
sku=config_azure['batch_pool_image_sku'])
vm_config = batchmodel.VirtualMachineConfiguration(
image_reference=image_reference,
node_agent_sku_id=config_azure['batch_pool_node_agent_sku'])
vm_start_task = batchmodel.StartTask(
command_line=
'/bin/bash -c "sudo yum -y install epel-release; sudo yum -y install python36 python36-devel python36-tools; sudo python36 -m ensurepip; sudo yum -y install openmpi openmpi-devel; sudo env MPICC=/usr/lib64/openmpi/bin/mpicc pip3 install mpi4py numpy; sudo pip3 --yes uninstall azure azure-common azure-storage; sudo pip3 install azure-storage azure-batch"',
user_identity=batchmodel.UserIdentity(
auto_user=batchmodel.AutoUserSpecification(
scope=batchmodel.AutoUserScope.pool,
elevation_level=batchmodel.ElevationLevel.admin)),
wait_for_success=True)
batch_service.pool.add(pool=batchmodel.PoolAddParameter(
id=config_azure['batch_pool_name'],
vm_size=config_azure['batch_pool_vm_size'],
virtual_machine_configuration=vm_config,
target_dedicated_nodes=config_azure[
'batch_pool_target_dedicated_nodes'],
enable_inter_node_communication=True,
start_task=vm_start_task),
raw=True)
def create_pool(batch_client,
name_pool,
number_nodes=0,
cmd_s_task=None,
rule_scale_pool=None):
#parameter image node
param_image = models.VirtualMachineConfiguration(
image_reference=models.ImageReference(offer='UbuntuServer',
publisher='Canonical',
sku='18.04-LTS',
version='latest',
virtual_machine_image_id=None),
node_agent_sku_id='batch.node.ubuntu 18.04')
#parameter pool
new_pool = models.PoolAddParameter(
id=name_pool,
vm_size='standard_d1_v2',
#target_dedicated_nodes = number_nodes,
virtual_machine_configuration=param_image,
enable_inter_node_communication=True,
enable_auto_scale=True,
auto_scale_formula=rule_scale_pool,
auto_scale_evaluation_interval='PT5M'
#start_task = s_task
)
batch_client.pool.add(new_pool)
def test_batch_create_pool_with_blobfuse_mount(self, **kwargs):
client = self.create_sharedkey_client(**kwargs)
# Test Create Iaas Pool
test_iaas_pool = models.PoolAddParameter(
id=self.get_resource_name('batch_iaas_'),
vm_size='Standard_A1',
virtual_machine_configuration=models.VirtualMachineConfiguration(
image_reference=models.ImageReference(
publisher='MicrosoftWindowsServer',
offer='WindowsServer',
sku='2016-Datacenter-smalldisk'
),
node_agent_sku_id='batch.node.windows amd64',
windows_configuration=models.WindowsConfiguration(enable_automatic_updates=True)),
task_scheduling_policy=models.TaskSchedulingPolicy(node_fill_type=models.ComputeNodeFillType.pack),
mount_configuration=[models.MountConfiguration(
azure_blob_file_system_configuration=models.AzureBlobFileSystemConfiguration(
account_name='test',
container_name='https://test.blob.core.windows.net:443/test-container',
relative_mount_path='foo',
account_key='fake_account_key'
)
)]
)
response = client.pool.add(test_iaas_pool)
self.assertIsNone(response)
mount_pool = client.pool.get(test_iaas_pool.id)
self.assertIsNotNone(mount_pool.mount_configuration)
self.assertEqual(len(mount_pool.mount_configuration), 1)
self.assertIsNotNone(mount_pool.mount_configuration[0].azure_blob_file_system_configuration)
self.assertIsNone(mount_pool.mount_configuration[0].nfs_mount_configuration)
def __create_pool_and_job(self, cluster_conf: models.ClusterConfiguration, software_metadata_key: str, start_task, VmImageModel):
"""
Create a pool and job
:param cluster_conf: the configuration object used to create the cluster
:type cluster_conf: aztk.models.ClusterConfiguration
:parm software_metadata_key: the id of the software being used on the cluster
:param start_task: the start task for the cluster
:param VmImageModel: the type of image to provision for the cluster
:param wait: wait until the cluster is ready
"""
self._get_cluster_data(cluster_conf.cluster_id).save_cluster_config(cluster_conf)
# reuse pool_id as job_id
pool_id = cluster_conf.cluster_id
job_id = cluster_conf.cluster_id
# Get a verified node agent sku
sku_to_use, image_ref_to_use = \
helpers.select_latest_verified_vm_image_with_node_agent_sku(
VmImageModel.publisher, VmImageModel.offer, VmImageModel.sku, self.batch_client)
network_conf = None
if cluster_conf.subnet_id is not None:
network_conf = batch_models.NetworkConfiguration(
subnet_id=cluster_conf.subnet_id)
auto_scale_formula = "$TargetDedicatedNodes={0}; $TargetLowPriorityNodes={1}".format(
cluster_conf.vm_count, cluster_conf.vm_low_pri_count)
# Confiure the pool
pool = batch_models.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batch_models.VirtualMachineConfiguration(
image_reference=image_ref_to_use,
node_agent_sku_id=sku_to_use),
vm_size=cluster_conf.vm_size,
enable_auto_scale=True,
auto_scale_formula=auto_scale_formula,
auto_scale_evaluation_interval=timedelta(minutes=5),
start_task=start_task,
enable_inter_node_communication=True if not cluster_conf.subnet_id else False,
max_tasks_per_node=4,
network_configuration=network_conf,
metadata=[
batch_models.MetadataItem(
name=constants.AZTK_SOFTWARE_METADATA_KEY, value=software_metadata_key),
batch_models.MetadataItem(
name=constants.AZTK_MODE_METADATA_KEY, value=constants.AZTK_CLUSTER_MODE_METADATA)
])
# Create the pool + create user for the pool
helpers.create_pool_if_not_exist(pool, self.batch_client)
# Create job
job = batch_models.JobAddParameter(
id=job_id,
pool_info=batch_models.PoolInformation(pool_id=pool_id))
# Add job to batch
self.batch_client.job.add(job)
return helpers.get_cluster(cluster_conf.cluster_id, self.batch_client)
def create_pool(self, size, name):
"""Create and deploy a new pool.
Called on job submission by submission.py.
TODO: Support auto-scale formula.
"""
image = self.environment.get_image()
node_agent_sku_id = image.pop('node_sku_id')
pool_id = 'Maya_Pool_{}'.format(uuid.uuid4())
pool_config = models.VirtualMachineConfiguration(
image_reference=models.ImageReference(**image),
node_agent_sku_id=node_agent_sku_id)
self._log.info("Creating new pool '{}' with {} VMs.".format(
name, size))
new_pool = models.PoolAddParameter(
id=pool_id,
display_name="Maya Pool for {}".format(name),
resize_timeout=datetime.timedelta(minutes=30),
application_licenses=self.environment.get_application_licenses(),
vm_size=self.environment.get_vm_sku(),
virtual_machine_configuration=pool_config,
target_dedicated_nodes=int(size[0]),
target_low_priority_nodes=int(size[1]),
max_tasks_per_node=1)
self._call(self.batch.pool.add, new_pool)
self._log.debug("Successfully created pool.")
return {"poolId": pool_id}
def create_pool(batch_client, pool_id, vm_size, vm_count, app_files):
"""Creates an Azure Batch pool with the specified id.
:param batch_client: The batch client to use.
:type batch_client: `batchserviceclient.BatchServiceClient`
:param block_blob_client: The storage block blob client to use.
:type block_blob_client: `azure.storage.blob.BlockBlobService`
:param str pool_id: The id of the pool to create.
:param str vm_size: vm size (sku)
:param int vm_count: number of vms to allocate
:param list app_files: The list of all the other scripts to upload.
"""
# pick the latest supported 16.04 sku for UbuntuServer
sku_to_use, image_ref_to_use = \
common.helpers.select_latest_verified_vm_image_with_node_agent_sku(
batch_client, 'Canonical', 'UbuntuServer', '14.04')
user = batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)
task_commands = get_list_from_file('configs/start_commands')
print(task_commands)
pool = batchmodels.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=image_ref_to_use, node_agent_sku_id=sku_to_use),
vm_size=vm_size,
target_dedicated=vm_count,
start_task=batchmodels.StartTask(
command_line=common.helpers.wrap_commands_in_shell(
'linux', task_commands),
user_identity=batchmodels.UserIdentity(auto_user=user),
resource_files=app_files,
wait_for_success=True))
common.helpers.create_pool_if_not_exist(batch_client, pool)
def create_pool(batch_service_client: batch.BatchServiceClient, config: Dict[str, str]) -> None:
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param config: Configuration
"""
print("Creating pool [{}]...".format(config['POOL_ID']))
# Create a new pool of Linux compute nodes using an Azure Virtual Machines
# Marketplace image. For more information about creating pools of Linux
# nodes, see:
# https://azure.microsoft.com/documentation/articles/batch-linux-nodes/
new_pool = batch.models.PoolAddParameter(
id=config['POOL_ID'],
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=batchmodels.ImageReference(
publisher="microsoft-azure-batch",
offer="ubuntu-server-container",
sku='16-04-lts',
version="latest",
),
node_agent_sku_id="batch.node.ubuntu 16.04",
container_configuration=create_container_config(config),
),
vm_size=config['POOL_VM_SIZE'],
target_dedicated_nodes=config['POOL_NODE_COUNT'],
task_slots_per_node=config['TASK_SLOTS_PER_NODE'],
)
batch_service_client.pool.add(new_pool)
def create_pool(batch_service_client, pool_id, vm_size, imageName, versions, auto_scale_formula):
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str pool_id: An ID for the new pool.
:param str publisher: Marketplace image publisher
:param str offer: Marketplace image offer
:param str sku: Marketplace image sky
"""
print('Creating pool [{}]...'.format(pool_id))
new_pool = batch.models.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=batchmodels.ImageReference(
virtual_machine_image_id="/subscriptions/{}/resourceGroups/{}/providers/Microsoft.Compute/images/{}".format(
"ad49354a-6ce2-4dae-a51d-b6907372f608", "BrowseCloud", imageName)
),
node_agent_sku_id="batch.node.windows amd64"),
vm_size=vm_size,
start_task=None,
enable_auto_scale=True,
auto_scale_formula=auto_scale_formula,
application_package_references=[batchmodels.ApplicationPackageReference(
application_id="browsecloudtrainer", version=version) for version in versions],
auto_scale_evaluation_interval=timedelta(
minutes=5) # the smallest evaluation interval
)
batch_service_client.pool.add(new_pool)
def create_pool(batch_client, block_blob_client, pool_id, vm_size, vm_count):
"""Creates an Azure Batch pool with the specified id.
:param batch_client: The batch client to use.
:type batch_client: `batchserviceclient.BatchServiceClient`
:param block_blob_client: The storage block blob client to use.
:type block_blob_client: `azure.storage.blob.BlockBlobService`
:param str pool_id: The id of the pool to create.
:param str vm_size: vm size (sku)
:param int vm_count: number of vms to allocate
"""
#application_packages = [
# batchmodels.ApplicationPackageReference(application_id='Regridv3', version='1.0.0'),
# batchmodels.ApplicationPackageReference(application_id='us_5km', version='1.0.0'),
# batchmodels.ApplicationPackageReference(application_id='us_800m_t6', version='1.0.0'),
#]
sku_to_use, image_ref_to_use = \
common.helpers.select_latest_verified_vm_image_with_node_agent_sku(
batch_client, 'Canonical', 'UbuntuServer', '18.04')
#image_ref_to_use = batchmodels.ImageReference(
# publisher = 'microsoftwindowsserver', offer = 'windowsserver', sku = '2019-datacenter', version = 'latest'
#)
#sku_to_use = "batch.node.windows amd64"
pool = batchmodels.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=image_ref_to_use, node_agent_sku_id=sku_to_use),
vm_size=vm_size,
max_tasks_per_node=_POOL_MAX_TASKS_PER_NODE,
#application_package_references=application_packages,
target_dedicated_nodes=vm_count)
common.helpers.create_pool_if_not_exist(batch_client, pool)
def create_pool(batch_service_client, pool_id):
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str pool_id: An ID for the new pool.
:param str publisher: Marketplace image publisher
:param str offer: Marketplace image offer
:param str sku: Marketplace image sku
"""
print('Creating pool [{}]...'.format(pool_id))
# Create a new pool of Linux compute nodes using an Azure Virtual Machines
# Marketplace image. For more information about creating pools of Linux
# nodes, see:
# https://azure.microsoft.com/documentation/articles/batch-linux-nodes/
new_pool = batch.models.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=batchmodels.ImageReference(publisher="Canonical",
offer="UbuntuServer",
sku="16.04-LTS",
version="latest"),
node_agent_sku_id="batch.node.ubuntu 16.04"),
vm_size=config._POOL_VM_SIZE,
target_dedicated_nodes=config._POOL_NODE_COUNT)
batch_service_client.pool.add(new_pool)
def create_pool(pool_id, batch_service_client=None):
"""
Creates a pool of compute nodes.
Parameters
==========
pool_id: str, identifier for the pool
batch_service_client: azure.batch.BatchServiceClient, A Batch service client.
"""
print("Creating pool [{}]...".format(pool_id))
if not batch_service_client:
batch_service_client = create_batch_client()
new_pool = batch.models.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=batchmodels.ImageReference(
publisher="Canonical",
offer="UbuntuServer",
sku="18.04-LTS",
version="latest",
),
node_agent_sku_id="batch.node.ubuntu 18.04",
),
vm_size=config["pool_vm_size"],
target_low_priority_nodes=config["pool_low_priority_node_count"],
target_dedicated_nodes=config["pool_dedicated_node_count"],
)
batch_service_client.pool.add(new_pool)
def create_pool_and_wait_for_nodes(batch_client, block_blob_client, pool_id,
vm_size, vm_count):
"""Creates an Azure Batch pool with the specified id.
:param batch_client: The batch client to use.
:type batch_client: `batchserviceclient.BatchServiceClient`
:param block_blob_client: The storage block blob client to use.
:type block_blob_client: `azure.storage.blob.BlockBlobService`
:param str pool_id: The id of the pool to create.
:param str vm_size: vm size (sku)
:param int vm_count: number of vms to allocate
:rtype: list
:return: list of `batchserviceclient.models.ComputeNode`
"""
# pick the latest supported 14.04 sku for UbuntuServer
sku_to_use, image_ref_to_use = \
common.helpers.select_latest_verified_vm_image_with_node_agent_sku(
batch_client, 'Canonical', 'UbuntuServer', '14.04')
# upload start task script
block_blob_client.create_container(_CONTAINER_NAME, fail_on_exist=False)
sas_url = common.helpers.upload_blob_and_create_sas(
block_blob_client, _CONTAINER_NAME, _STARTTASK_RESOURCE_FILE,
_STARTTASK_SHELL_SCRIPT_PATH,
datetime.datetime.utcnow() + datetime.timedelta(hours=1))
# create pool with start task
pool = batchmodels.CloudPool(
id=pool_id,
enable_inter_node_communication=True,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=image_ref_to_use, node_agent_sku_id=sku_to_use),
vm_size=vm_size,
target_dedicated=vm_count,
start_task=batchmodels.StartTask(
command_line=_STARTTASK_RESOURCE_FILE,
run_elevated=True,
wait_for_success=True,
resource_files=[
batchmodels.ResourceFile(file_path=_STARTTASK_RESOURCE_FILE,
blob_source=sas_url)
]),
)
common.helpers.create_pool_if_not_exist(batch_client, pool)
# because we want all nodes to be available before any tasks are assigned
# to the pool, here we will wait for all compute nodes to reach idle
nodes = common.helpers.wait_for_all_nodes_state(
batch_client, pool,
frozenset((batchmodels.ComputeNodeState.starttaskfailed,
batchmodels.ComputeNodeState.unusable,
batchmodels.ComputeNodeState.idle)))
# ensure all node are idle
if any(node.state != batchmodels.ComputeNodeState.idle for node in nodes):
raise RuntimeError('node(s) of pool {} not in idle state'.format(
pool.id))
return nodes
def create_pool(batch_service_client, pool_id, users):
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str pool_id: An ID for the new pool.
:param str publisher: Marketplace image publisher
:param str offer: Marketplace image offer
:param str sku: Marketplace image sky
"""
print('Creating pool [{}]...'.format(pool_id))
# Create a new pool of Linux compute nodes using an Azure Virtual Machines
# Marketplace image. For more information about creating pools of Linux
# nodes, see:
# https://azure.microsoft.com/documentation/articles/batch-linux-nodes/
# The start task installs ffmpeg on each node from an available repository, using
# an administrator user identity.
new_pool = batch.models.PoolAddParameter(
id=pool_id,
user_accounts=users,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=batchmodels.ImageReference(publisher="Canonical",
offer="UbuntuServer",
sku="18.04-LTS",
version="latest"),
node_agent_sku_id="batch.node.ubuntu 18.04"),
vm_size=_POOL_VM_SIZE,
target_dedicated_nodes=_DEDICATED_POOL_NODE_COUNT,
target_low_priority_nodes=_LOW_PRIORITY_POOL_NODE_COUNT,
start_task=batchmodels.StartTask(
#command_line="/bin/bash -c \"git clone https://github.com/uiuc-arc/probfuzz.git; \
# cd probfuzz/; ./install_java.sh; ./install.sh\"",
# command_line="/bin/bash -c \"apt-get update\"",
command_line="/bin/bash -c \"sudo apt-get -y update; \
sudo apt-get install git; \
sudo apt-get install -y python2.7; \
sudo apt-get install -y python-pip; \
sudo apt-get install -y bc; \
sudo apt-get install -y r-base; \
sudo pip2 --no-cache-dir install pandas; \
sudo pip2 install rpy2==2.8.6;\
sudo pip2 install argparse;\
sudo pip2 install numpy;\
sudo pip2 install scipy;\
\"",
wait_for_success=True,
user_identity=batchmodels.UserIdentity(user_name="admin"),
#user_identity=batchmodels.UserIdentity(
# auto_user=batchmodels.AutoUserSpecification(
# scope=batchmodels.AutoUserScope.pool,
# elevation_level=batchmodels.ElevationLevel.admin)),
),
max_tasks_per_node=_MAX_TASKS_PER_NODE)
batch_service_client.pool.add(new_pool)
def create_pool(batch_service_client, pool_id):
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str pool_id: An ID for the new pool.
:param str publisher: Marketplace image publisher
:param str offer: Marketplace image offer
:param str sku: Marketplace image sky
"""
print('Creating pool [{}]...'.format(pool_id))
# Create a new pool of Linux compute nodes using an Azure Virtual Machines
# Marketplace image. For more information about creating pools of Linux
# nodes, see:
# https://azure.microsoft.com/documentation/articles/batch-linux-nodes/
# The start task installs ffmpeg on each node from an available repository, using
# an administrator user identity.
#
task_commands = [
"apt-get update",
"apt-get -y install python3-pip",
"apt -y install htop",
"apt -y install iftop",
"pip3 install azure-storage-blob",
"pip3 install pyspark",
"pip3 install pandas",
"apt -y install openjdk-8-jre-headless"
]
new_pool = batch.models.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=batchmodels.ImageReference(
publisher="Canonical",
offer="UbuntuServer",
sku="18.04-LTS",
version="latest"
),
node_agent_sku_id="batch.node.ubuntu 18.04"),
vm_size=POOL_VM_SIZE,
target_dedicated_nodes=DEDICATED_POOL_NODE_COUNT,
target_low_priority_nodes=LOW_PRIORITY_POOL_NODE_COUNT,
start_task=batchmodels.StartTask(
command_line=wrap_commands_in_shell('linux',task_commands),
wait_for_success=True,
user_identity=batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)),
)
)
batch_service_client.pool.add(new_pool)
def create_pool(batch_service_client, pool_id,
resource_files, publisher, offer, sku,
vm_size, node_count):
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str pool_id: An ID for the new pool.
:param list resource_files: A collection of resource files for the pool's
start task.
:param str publisher: Marketplace image publisher
:param str offer: Marketplace image offer
:param str sku: Marketplace image sku
"""
print('Creating pool [{}]...'.format(pool_id))
# Create a new pool of Linux compute nodes using an Azure Virtual Machines
# Marketplace image. For more information about creating pools of Linux
# nodes, see:
# https://azure.microsoft.com/documentation/articles/batch-linux-nodes/
# Specify the commands for the pool's start task. The start task is run
# on each node as it joins the pool, and when it's rebooted or re-imaged.
# If there were installs needed, this is where we'd add them as
task_commands = ['echo starting up']
# Get the node agent SKU and image reference for the virtual machine
# configuration.
# For more information about the virtual machine configuration, see:
# https://azure.microsoft.com/documentation/articles/batch-linux-nodes/
sku_to_use, image_ref_to_use = \
select_latest_verified_vm_image_with_node_agent_sku(
batch_service_client, publisher, offer, sku)
user = batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)
new_pool = batch.models.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=image_ref_to_use,
node_agent_sku_id=sku_to_use),
vm_size=vm_size,
target_dedicated_nodes=node_count,
start_task=batch.models.StartTask(
command_line=wrap_commands_in_shell('linux',task_commands),
user_identity=batchmodels.UserIdentity(auto_user=user),
wait_for_success=True,
resource_files=resource_files),
)
try:
batch_service_client.pool.add(new_pool)
except batchmodels.batch_error.BatchErrorException as err:
print(err)
raise
def create_pool(self, pool_id):
"""
Creates a pool of compute nodes with the specified OS settings.
:param str pool_id: An ID for the new pool.
:param dict config: Configuration details.
"""
if pool_id in self.active_pools or self.pool_exists(pool_id):
return
self.logger.info("creating pool {}".format(pool_id))
pool_config = self.config['pools'][pool_id]
sku_to_use, image_ref_to_use = self.__get_vm_image_and_node_agent_sku(
pool_config)
start_vm_commands = None
if pool_config.get('create_vm_commands', None):
start_vm_commands = self.__create_commands(
pool_config['create_vm_commands'])
user = batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)
vm_configuration = batchmodels.VirtualMachineConfiguration(
image_reference=image_ref_to_use,
node_agent_sku_id=sku_to_use,
)
vm_start_task = batchmodels.StartTask(
command_line=self.__wrap_commands_in_shell('linux',
start_vm_commands),
user_identity=batchmodels.UserIdentity(auto_user=user),
wait_for_success=True)
new_pool = batchmodels.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=vm_configuration,
vm_size=pool_config['pool_vm_size'],
enable_auto_scale=True,
auto_scale_formula=pool_config['auto_scale_formula'],
auto_scale_evaluation_interval=datetime.timedelta(minutes=5),
start_task=vm_start_task,
max_tasks_per_node=pool_config['max_tasks_per_node'],
)
try:
self.batch_client.pool.add(new_pool)
except batchmodels.BatchErrorException as err:
self.__print_batch_exception(err)
raise
self.active_pools.add(pool_id)
def test_batch_network_configuration(self, **kwargs):
client = self.create_aad_client(**kwargs)
# Test Create Pool with Network Config
network_config = models.NetworkConfiguration(
endpoint_configuration=models.PoolEndpointConfiguration(
inbound_nat_pools=[
models.InboundNATPool(
name="TestEndpointConfig",
protocol=models.InboundEndpointProtocol.udp,
backend_port=64444,
frontend_port_range_start=60000,
frontend_port_range_end=61000,
network_security_group_rules=[
models.NetworkSecurityGroupRule(
priority=150,
access=models.NetworkSecurityGroupRuleAccess.allow,
source_address_prefix='*'
)
]
)
]
)
)
virtual_machine_config = models.VirtualMachineConfiguration(
node_agent_sku_id="batch.node.ubuntu 16.04",
image_reference=models.ImageReference(
publisher="Canonical",
offer="UbuntuServer",
sku="16.04-LTS")
)
pool = models.PoolAddParameter(
id=self.get_resource_name('batch_network_'),
target_dedicated_nodes=1,
vm_size='Standard_A1',
virtual_machine_configuration=virtual_machine_config,
network_configuration=network_config
)
client.pool.add(pool)
network_pool = client.pool.get(pool.id)
while self.is_live and network_pool.allocation_state != models.AllocationState.steady:
time.sleep(10)
network_pool = client.pool.get(pool.id)
# Test Compute Node Config
nodes = list(client.compute_node.list(pool.id))
self.assertEqual(len(nodes), 1)
self.assertIsInstance(nodes[0], models.ComputeNode)
self.assertEqual(len(nodes[0].endpoint_configuration.inbound_endpoints), 2)
self.assertEqual(nodes[0].endpoint_configuration.inbound_endpoints[0].name, 'TestEndpointConfig.0')
self.assertEqual(nodes[0].endpoint_configuration.inbound_endpoints[0].protocol.value, 'udp')
def create_pool(batch_service_client, pool_id):
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str pool_id: An ID for the new pool.
:param str publisher: Marketplace image publisher
:param str offer: Marketplace image offer
:param str sku: Marketplace image sku
"""
print('Creating pool [{}]...'.format(pool_id))
# Create a new pool of Linux compute nodes using an Azure Virtual Machines
# Marketplace image. For more information about creating pools of Linux
# nodes, see:
# https://azure.microsoft.com/documentation/articles/batch-linux-nodes/
# In this case is a pool ready to run Docker containers
image_ref_to_use = batch.models.ImageReference(
publisher='microsoft-azure-batch',
offer='ubuntu-server-container',
sku='16-04-lts',
version='latest'
)
# Specify a container registry
# We got the credentials from config.py
containerRegistry = batchmodels.ContainerRegistry(
user_name=config._REGISTRY_USER_NAME,
password=config._REGISTRY_PASSWORD,
registry_server=config._REGISTRY_SERVER
)
# The instance will pull the images defined here
container_conf = batchmodels.ContainerConfiguration(
container_image_names=[config._DOCKER_IMAGE],
container_registries=[containerRegistry]
)
new_pool = batch.models.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=image_ref_to_use,
container_configuration=container_conf,
node_agent_sku_id='batch.node.ubuntu 16.04'),
vm_size=config._POOL_VM_SIZE,
target_dedicated_nodes=config._POOL_NODE_COUNT
)
batch_service_client.pool.add(new_pool)
def get_virtual_machine_configuration(self):
if self.image_id:
image_ref = batchmodels.ImageReference(
virtual_machine_image_id=self.image_id)
else:
image_ref = batchmodels.ImageReference(
publisher=self.image_publisher,
offer=self.image_offer,
sku=self.image_sku,
version=self.image_version)
return batchmodels.VirtualMachineConfiguration(
image_reference=image_ref,
node_agent_sku_id=self.node_agent_sku_id)
def create_pool_and_wait_for_node(batch_client, pool_id, vm_size, vm_count,
sha1_cert_tp):
"""Creates an Azure Batch pool with the specified id.
:param batch_client: The batch client to use.
:type batch_client: `batchserviceclient.BatchServiceClient`
:param str pool_id: The id of the pool to create.
:param str vm_size: vm size (sku)
:param int vm_count: number of vms to allocate
:param str sha1_cert_tp: sha1 cert thumbprint for cert ref
"""
# pick the latest supported 16.04 sku for UbuntuServer
sku_to_use, image_ref_to_use = \
common.helpers.select_latest_verified_vm_image_with_node_agent_sku(
batch_client, 'Canonical', 'UbuntuServer', '16.04')
# create start task commands
# 1. update repository
# 2. install blobxfer pre-requisites
# 3. pip install blobxfer python script
start_task_commands = [
'apt-get update',
'apt-get install -y build-essential libssl-dev libffi-dev ' +
'libpython-dev python-dev python-pip', 'pip install --upgrade blobxfer'
]
user = batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)
# create pool with start task and cert ref with visibility of task
pool = batchmodels.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=image_ref_to_use, node_agent_sku_id=sku_to_use),
vm_size=vm_size,
target_dedicated=vm_count,
start_task=batchmodels.StartTask(
command_line=common.helpers.wrap_commands_in_shell(
'linux', start_task_commands),
user_identity=batchmodels.UserIdentity(auto_user=user),
wait_for_success=True),
certificate_references=[
batchmodels.CertificateReference(
sha1_cert_tp,
'sha1',
visibility=[batchmodels.CertificateVisibility.task])
],
)
common.helpers.create_pool_if_not_exist(batch_client, pool)
def create_processing_pool(batch_service_client, start_task):
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:param str start_task: task start command.
:type batch_service_client: `azure.batch.BatchServiceClient`
"""
LOGGER.info(f'Creating pool [{PROCESSING_POOL_ID}]...')
image_ref_to_use = get_image_reference()
container_registry = \
batch_models.ContainerRegistry(
registry_server=REGISTRY_SERVER,
user_name=REGISTRY_ACCOUNT_USER,
password=REGISTRY_ACCOUNT_PASSWORD)
container_conf = batch_models.ContainerConfiguration(
container_image_names=[DOCKER_CONTAINER_URL],
container_registries=[container_registry])
new_pool = batch_models.PoolAddParameter(
id=PROCESSING_POOL_ID,
virtual_machine_configuration=
batch_models.VirtualMachineConfiguration(
image_reference=image_ref_to_use,
container_configuration=container_conf,
node_agent_sku_id=VM_AGENT_SKU),
vm_size=PROCESSING_POOL_VM_SIZE,
start_task=batch_models.StartTask(
command_line=start_task,
user_identity=batch_models.UserIdentity(
auto_user=batch_models.AutoUserSpecification(
scope='pool',
elevation_level='admin'))
),
enable_auto_scale=True,
auto_scale_evaluation_interval=datetime.timedelta(
minutes=PROCESSING_POOL_SCALE_INTERVAL_MINUTES),
auto_scale_formula=PROCESSING_POOL_SCALE_FORMULA)
try:
batch_service_client.pool.add(new_pool)
LOGGER.info("Processing Pool Created")
except batch_models.BatchErrorException as err:
if 'The specified pool already exists.' in err.error.message.value:
LOGGER.info("Pool already exists...")
else:
raise
def __create_pool_and_job(self, cluster_conf, software_metadata_key: str, start_task, VmImageModel):
"""
Create a pool and job
:param cluster_conf: the configuration object used to create the cluster
:type cluster_conf: aztk.models.ClusterConfiguration
:parm software_metadata_key: the id of the software being used on the cluster
:param start_task: the start task for the cluster
:param VmImageModel: the type of image to provision for the cluster
:param wait: wait until the cluster is ready
"""
# reuse pool_id as job_id
pool_id = cluster_conf.cluster_id
job_id = cluster_conf.cluster_id
# Get a verified node agent sku
sku_to_use, image_ref_to_use = \
helpers.select_latest_verified_vm_image_with_node_agent_sku(
VmImageModel.publisher, VmImageModel.offer, VmImageModel.sku, self.batch_client)
# Confiure the pool
pool = batch_models.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batch_models.VirtualMachineConfiguration(
image_reference=image_ref_to_use,
node_agent_sku_id=sku_to_use),
vm_size=cluster_conf.vm_size,
target_dedicated_nodes=cluster_conf.vm_count,
target_low_priority_nodes=cluster_conf.vm_low_pri_count,
start_task=start_task,
enable_inter_node_communication=True,
max_tasks_per_node=1,
metadata=[
batch_models.MetadataItem(
name=constants.AZTK_SOFTWARE_METADATA_KEY, value=software_metadata_key),
])
# Create the pool + create user for the pool
helpers.create_pool_if_not_exist(pool, self.batch_client)
# Create job
job = batch_models.JobAddParameter(
id=job_id,
pool_info=batch_models.PoolInformation(pool_id=pool_id))
# Add job to batch
self.batch_client.job.add(job)
return helpers.get_cluster(cluster_conf.cluster_id, self.batch_client)
def create_pool(config, batch_service_client):
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str pool_id: An ID for the new pool.
:param str publisher: Marketplace image publisher
:param str offer: Marketplace image offer
:param str sku: Marketplace image sku
"""
# Create a new pool of Linux compute nodes using an Azure Virtual Machines
# Marketplace image. For more information about creating pools of Linux
# nodes, see:
# https://azure.microsoft.com/documentation/articles/batch-linux-nodes/
image_ref_to_use = models.ImageReference(
publisher="microsoft-azure-batch",
offer="ubuntu-server-container",
sku="16-04-lts",
version="latest",
)
if config.REGISTRY_USERNAME:
registry = models.ContainerRegistry(
user_name=config.REGISTRY_USERNAME,
password=config.REGISTRY_PASSWORD,
registry_server=config.REGISTRY_SERVER,
)
container_conf = models.ContainerConfiguration(
container_image_names=[config.DOCKER_CONTAINER],
container_registries=[registry],
)
else:
container_conf = models.ContainerConfiguration(
container_image_names=[config.DOCKER_CONTAINER])
new_pool = models.PoolAddParameter(
id=config.POOL_ID,
virtual_machine_configuration=models.VirtualMachineConfiguration(
image_reference=image_ref_to_use,
container_configuration=container_conf,
node_agent_sku_id="batch.node.ubuntu 16.04",
),
vm_size=config.POOL_VM_SIZE,
target_dedicated_nodes=config.POOL_NODE_COUNT,
target_low_priority_nodes=config.POOL_LOW_PRIORITY_NODE_COUNT,
)
batch_service_client.pool.add(new_pool)
def create_pool(batch_service_client, pool_id):
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str pool_id: An ID for the new pool.
:param str publisher: Marketplace image publisher
:param str offer: Marketplace image offer
:param str sku: Marketplace image sky
"""
print('Creating pool [{}]...'.format(pool_id))
# Create a new pool of Linux compute nodes using an Azure Virtual Machines
# Marketplace image. For more information about creating pools of Linux
# nodes, see:
# https://azure.microsoft.com/documentation/articles/batch-linux-nodes/
# The start task installs ffmpeg on each node from an available repository, using
# an administrator user identity.
new_pool = batch.models.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=batchmodels.ImageReference(publisher="Canonical",
offer="UbuntuServer",
sku="16.04-LTS",
version="latest"),
node_agent_sku_id="batch.node.ubuntu 16.04"),
vm_size=os.environ['POOL_VM_SIZE'],
target_dedicated_nodes=os.environ['DEDICATED_POOL_NODE_COUNT'],
target_low_priority_nodes=os.environ['LOW_PRIORITY_POOL_NODE_COUNT'],
start_task=batchmodels.StartTask(
command_line="/bin/bash -c \"apt-get update && \
apt-get install -y apt-transport-https ca-certificates curl software-properties-common &&\
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \
add-apt-repository 'deb [arch=amd64] https://download.docker.com/linux/ubuntu xenial stable' && \
apt-get update && \
apt-get install -y docker-ce\"",
wait_for_success=True,
user_identity=batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)),
))
batch_service_client.pool.add(new_pool)
def create_pool(batch_service_client, pool_id,
publisher, offer, sku, pool_vm_size, pool_node_count, low_priority_node_count, task_commands):
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str pool_id: An ID for the new pool.
:param list resource_files: A collection of resource files for the pool's
start task.
:param str publisher: Marketplace image publisher
:param str offer: Marketplace image offer
:param str sku: Marketplace image sku
"""
print('Creating pool [{}]...'.format(pool_id))
# Get the node agent SKU and image reference for the virtual machine
# configuration.
# For more information about the virtual machine configuration, see:
# https://azure.microsoft.com/documentation/articles/batch-linux-nodes/
sku_to_use, image_ref_to_use = \
common.helpers.select_latest_verified_vm_image_with_node_agent_sku(
batch_service_client, publisher, offer, sku)
user = batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)
new_pool = batch.models.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=image_ref_to_use,
node_agent_sku_id=sku_to_use),
vm_size=pool_vm_size,
target_dedicated_nodes=pool_node_count,
target_low_priority_nodes=low_priority_node_count,
start_task=batch.models.StartTask(
command_line=common.helpers.wrap_commands_in_shell('linux',
task_commands),
user_identity=batchmodels.UserIdentity(auto_user=user),
wait_for_success=True,
max_task_retry_count=-1)
)
try:
batch_service_client.pool.add(new_pool)
except batchmodels.batch_error.BatchErrorException as err:
print_batch_exception(err)
raise
def get_vm_config_for_distro(batch_service_client, distro, version):
"""
Gets a virtual machine configuration for the specified distro and version
from the list of Azure Virtual Machines Marketplace images verified to be
compatible with the Batch service.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str distro: The Linux distribution that should be installed on the
compute nodes, e.g. 'Ubuntu' or 'CentOS'. Supports partial string matching.
:param str version: The version of the operating system for the compute
nodes, e.g. '15' or '14.04'. Supports partial string matching.
:rtype: `azure.batch.models.VirtualMachineConfiguration`
:return: A virtual machine configuration specifying the Virtual Machines
Marketplace image and node agent SKU to install on the compute nodes in
a pool.
"""
# Get the list of node agents from the Batch service
node_agent_skus = batch_service_client.account.list_node_agent_skus()
# Get the first node agent that is compatible with the specified distro.
# Note that 'distro' in this case actually maps to the 'offer' property of
# the ImageReference.
node_agent = next(agent for agent in node_agent_skus
for image_ref in agent.verified_image_references
if distro.lower() in image_ref.offer.lower()
and version.lower() in image_ref.sku.lower())
# Get the last image reference from the list of verified references
# for the node agent we obtained. Typically, the verified image
# references are returned in ascending release order so this should give us
# the newest image.
img_ref = [
image_ref for image_ref in node_agent.verified_image_references
if distro.lower() in image_ref.offer.lower()
and version.lower() in image_ref.sku.lower()
][-1]
# Create the VirtualMachineConfiguration, specifying the VM image
# reference and the Batch node agent to be installed on the node.
# Note that these commands are valid for a pool of Ubuntu-based compute
# nodes, and that you may need to adjust the commands for execution
# on other distros.
vm_config = batchmodels.VirtualMachineConfiguration(
image_reference=img_ref, node_agent_sku_id=node_agent.id)
return vm_config
def create_pool_and_wait_for_nodes(batch_client, block_blob_client, pool_id,
vm_size, vm_count):
sku_to_use, image_ref_to_use = common.helpers.select_latest_verified_vm_image_with_node_agent_sku(
batch_client, 'Canonical', 'UbuntuServer', '14.04')
block_blob_client.create_container(_CONTAINER_NAME, fail_on_exist=False)
# upload start task script
block_blob_client.create_container(_CONTAINER_NAME, fail_on_exist=False)
sas_url = common.helpers.upload_blob_and_create_sas(
block_blob_client, _CONTAINER_NAME, _STARTTASK_RESOURCE_FILE,
_STARTTASK_SHELL_SCRIPT_PATH,
datetime.datetime.utcnow() + datetime.timedelta(hours=1))
# create pool and execute starttask
pool = batchmodels.PoolAddParameter(
id=pool_id,
enable_inter_node_communication=True,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=image_ref_to_use, node_agent_sku_id=sku_to_use),
vm_size=vm_size,
target_dedicated=vm_count,
start_task=batchmodels.StartTask(
command_line=_STARTTASK_RESOURCE_FILE,
run_elevated=True,
wait_for_success=True,
resource_files=[
batchmodels.ResourceFile(file_path=_STARTTASK_RESOURCE_FILE,
blob_source=sas_url)
]),
)
common.helpers.create_pool_if_not_exist(batch_client, pool)
# because we want all nodes to be available before any tasks are assigned
# to the pool, here we will wait for all compute nodes to reach idle
nodes = common.helpers.wait_for_all_nodes_state(
batch_client, pool,
frozenset((batchmodels.ComputeNodeState.starttaskfailed,
batchmodels.ComputeNodeState.unusable,
batchmodels.ComputeNodeState.idle)))
# ensure all node are idle
if any(node.state != batchmodels.ComputeNodeState.idle for node in nodes):
raise RuntimeError('node(s) of pool {} not in idle state'.format(
pool.id))
return nodes
def create_pool(batch_service_client, pool_id, application_files):
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str pool_id: An ID for the new pool.
:param str publisher: Marketplace image publisher
:param str offer: Marketplace image offer
:param str sku: Marketplace image sky
"""
print('Creating pool [{}]...'.format(pool_id))
# Create a new pool of Linux compute nodes using an Azure Virtual Machines
# Marketplace image. For more information about creating pools of Linux
# nodes, see:
# https://azure.microsoft.com/documentation/articles/batch-linux-nodes/
# The start task installs ffmpeg on each node from an available repository, using
# an administrator user identity.
new_pool = batch.models.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=batchmodels.ImageReference(
publisher="MicrosoftWindowsServer",
offer="WindowsServer",
sku="2016-Datacenter",
version="latest"
),
node_agent_sku_id="batch.node.windows amd64"),
vm_size=config._POOL_VM_SIZE,
target_dedicated_nodes=config._DEDICATED_POOL_NODE_COUNT,
target_low_priority_nodes=config._LOW_PRIORITY_POOL_NODE_COUNT,
start_task=batchmodels.StartTask(
command_line="cmd /c starttask.cmd",
resource_files=application_files,
wait_for_success=True,
user_identity=batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)),
)
)
batch_service_client.pool.add(new_pool)
def create_pool(batch_service_client, pool_id, resource_files):
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str pool_id: An ID for the new pool.
:param str publisher: Marketplace image publisher
:param str offer: Marketplace image offer
:param str sku: Marketplace image sku
"""
print('Creating pool [{}]...'.format(pool_id))
# Create a new pool of Linux compute nodes using an Azure Virtual Machines
# Marketplace image. For more information about creating pools of Linux
# nodes, see:
# https://azure.microsoft.com/documentation/articles/batch-linux-nodes/
task_commands = [
'cp -p {} $AZ_BATCH_NODE_SHARED_DIR'.format(config._TASK_FILE),
'''wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh && \
bash miniconda.sh -b -p $AZ_BATCH_NODE_SHARED_DIR/miniconda && \
export PATH="$AZ_BATCH_NODE_SHARED_DIR/miniconda/bin:$PATH" && \
source "$AZ_BATCH_NODE_SHARED_DIR/miniconda/bin/activate" && \
conda install -y -c anaconda -c conda-forge \
pip'''
]
image_ref = batchmodels.ImageReference(publisher="Canonical",
offer="UbuntuServer",
sku="18.04-LTS",
version="latest")
vm_config = batchmodels.VirtualMachineConfiguration(
image_reference=image_ref, node_agent_sku_id="batch.node.ubuntu 18.04")
new_pool = batch.models.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=vm_config,
vm_size=config._POOL_VM_SIZE,
target_low_priority_nodes=config._POOL_NODE_COUNT,
start_task=batch.models.StartTask(command_line=wrap_commands_in_shell(
'linux', task_commands),
resource_files=resource_files))
batch_service_client.pool.add(new_pool)
def create_pool(self):
from azure.batch import models as batchmodels
pool_id = self.compute_env_prefix + self.machine.name + '-' + str(
self.disk_size)
pool = self.get_pool(pool_id)
if pool is not None:
return pool_id
sku_to_use, image_ref_to_use = self.select_latest_verified_vm_image_with_node_agent_sku(
)
container_configuration = batchmodels.ContainerConfiguration(
container_image_names=[self.image])
config = batchmodels.VirtualMachineConfiguration(
image_reference=image_ref_to_use,
node_agent_sku_id=sku_to_use,
data_disks=[
batchmodels.DataDisk(disk_size_gb=self.disk_size, lun=1)
],
container_configuration=container_configuration,
)
pool = batchmodels.PoolAddParameter(
id=pool_id,
display_name=pool_id,
virtual_machine_configuration=config,
vm_size=self.machine.name,
)
if self.conf[utils.PLATFORM].get('low_priority', False):
pool.target_low_priority_nodes = 1
else:
pool.target_dedicated_nodes = 1
self.batch_client.pool.add(pool)
while self.get_pool(pool_id) is None:
time.sleep(1)
return pool_id