def create_pool(self, batch_service_client: batch.BatchExtensionsClient,
image_references: 'List[utils.ImageReference]'):
"""
Creates the Pool that will be submitted to the batch service.
:type batch_service_client: `azure.batch.BatchExtensionsClient`
:param batch_service_client: The batch client used for making batch operations
:type image_references: List['utils.ImageReference`]
:param image_references: A list of image references that job can run
"""
# load the template file
template = ctm.load_file(self.pool_template_file)
# set extra license if needed
if self.application_licenses is not None:
template["pool"][
"applicationLicenses"] = self.application_licenses.split(",")
# Set rendering version
ctm.set_image_reference(template, image_references)
ctm.set_template_pool_id(template, self.pool_id)
all_pools = [p.id for p in batch_service_client.pool.list()]
if self.pool_id not in all_pools:
self.submit_pool(batch_service_client, template)
else:
logger.info('pool [{}] already exists'.format(self.pool_id))
def submit_pool(self, batch_service_client: batch.BatchExtensionsClient,
template: str):
"""
Submits a batch pool based on the template
:param batch_service_client: The batch client used for making batch operations
:type batch_service_client: `azure.batch.BatchExtensionsClient`
:param template: The in memory version of the template used to create a the job.
:type template: str
"""
parameters = ctm.load_file(self.parameters_file)
#updates any placeholder parameter values with the values from keyVault, if required
utils.update_params_with_values_from_keyvault(
parameters, self.keyvault_client_with_url)
pool_json = batch_service_client.pool.expand_template(
template, parameters)
ctm.set_template_pool_id(template, self.pool_id)
pool = batch_service_client.pool.poolparameter_from_json(pool_json)
logger.info('Creating pool [{}]...'.format(pool))
try:
batch_service_client.pool.add(pool)
except batchmodels.BatchErrorException as err:
if utils.expected_exception(err,
"The specified pool already exists"):
logger.warning("Pool [{}] is already being created.".format(
self.pool_id))
else:
logger.info("Create pool error: {}".format(err))
traceback.print_exc()
utils.print_batch_exception(err)
def create_and_submit_job(self, batch_client: batch.BatchExtensionsClient):
"""
Creates the Job that will be submitted to the batch service
:param batch_client: The batch client to use.
:type batch_client: `azure.batch.BatchExtensionsClient`
"""
logger.info('Creating Job [{}]... job will run on [{}]'.format(
self.job_id, self.pool_id))
# load the template and parameters file
template = ctm.load_file(self.template_file)
parameters = ctm.load_file(self.parameters_file)
#updates any placeholder parameter values with the values from keyVault, if required
utils.update_params_with_values_from_keyvault(
parameters, self.keyvault_client_with_url)
# overrides some of the parameters needed in the file, container SAS
# tokens need to be generated for the container
ctm.set_parameter_name(parameters, self.job_id)
ctm.set_parameter_storage_info(parameters, self.storage_info)
ctm.set_template_pool_id(parameters, self.pool_id)
ctm.set_job_resource_file_urls_to_branch(template,
self.repository_branch_name)
# Submits the job
self.submit_job(batch_client, template, parameters)
def submit_pool(self, batch_service_client: batch.BatchExtensionsClient, template: str):
"""
Submits a batch pool based on the template
:param batch_service_client: The batch client used for making batch operations
:type batch_service_client: `azure.batch.BatchExtensionsClient`
:param template: The in memory version of the template used to create a the job.
:type template: str
"""
parameters = ctm.load_file(self.parameters_file)
pool_json = batch_service_client.pool.expand_template(template, parameters)
ctm.set_template_pool_id(template, self.pool_id)
pool = batch_service_client.pool.poolparameter_from_json(pool_json)
logger.info('Creating pool [{}]...'.format(self.pool_id))
try:
batch_service_client.pool.add(pool)
except batchmodels.batch_error.BatchErrorException as err:
if utils.expected_exception(
err, "The specified pool already exists"):
logger.warning(
"Pool [{}] is already being created.".format(
self.pool_id))
else:
logger.info("Create pool error: {}".format(err))
traceback.print_exc()
utils.print_batch_exception(err)
def create_pool(self,
batch_service_client: batch.BatchExtensionsClient,
image_references: 'List[utils.ImageReference]',
VM_image_URL=None,
VM_OS_type=None):
"""
Creates the Pool that will be submitted to the batch service.
:type batch_service_client: `azure.batch.BatchExtensionsClient`
:param batch_service_client: The batch client used for making batch operations
:type image_references: List['utils.ImageReference`]
:param image_references: A list of image references that job can run
:type VM_image_URL: str
:param VM_image_URL: The resource link to an image inside your image repo. If this is resource link is provided
the VMs will use the custom image you provided.
:type VM_OS_type: str
:param VM_OS_type: The custom image operating system type, this can be windows or centos. This is needed if you
want to use a custom image.
"""
# load the template file
template = ctm.load_file(self.pool_template_file)
# set extra license if needed
if self.application_licenses is not None:
template["pool"][
"applicationLicenses"] = self.application_licenses.split(",")
# Set rendering version
ctm.set_image_reference(template, image_references)
ctm.set_template_pool_id(template, self.pool_id)
ctm.set_pool_resource_file_urls_to_branch(template,
self.repository_branch_name)
if VM_image_URL is not None:
ctm.set_custom_image(template, VM_image_URL, VM_OS_type)
all_pools = [p.id for p in batch_service_client.pool.list()]
if self.pool_id not in all_pools:
self.submit_pool(batch_service_client, template)
else:
logger.info('pool [{}] already exists'.format(self.pool_id))
def submit_pool(self, batch_service_client: batch.BatchExtensionsClient, template: str, UseLowPriorityVMs: bool):
"""
Submits a batch pool based on the template
:param batch_service_client: The batch client used for making batch operations
:type batch_service_client: `azure.batch.BatchExtensionsClient`
:param template: The in memory version of the template used to create a the job.
:type template: str
:param UseLowPriorityVMs: True: Use low priority nodes, False: will use the type in the template file
:type UseLowPriorityVMs: bool
"""
parameters = ctm.load_file(self.parameters_file)
if UseLowPriorityVMs==True:
#swap the dedicated vm count to low pri and zero out the dedicated count (to reduce testing COGS)
ctm.set_low_priority_vm_count(template, str(self.min_required_vms))
#some tests set the dedicated vm count in the test parameters file, and some use the default in the template, so override both to 0
ctm.set_dedicated_vm_count(parameters, 0)
ctm.set_dedicated_vm_count(template, 0)
# updates any placeholder parameter values with the values from
# keyVault, if required
utils.update_params_with_values_from_keyvault(
parameters, self.keyvault_client_with_url)
pool_json = batch_service_client.pool.expand_template(
template, parameters)
ctm.set_template_pool_id(template, self.pool_id)
pool = batch_service_client.pool.poolparameter_from_json(pool_json)
logger.info('Creating pool [{}]...'.format(pool))
try:
utils.run_with_jitter_retry(batch_service_client.pool.add, pool)
except batchmodels.BatchErrorException as err:
if utils.expected_exception(
err, "The specified pool already exists"):
logger.warning(
"Pool [{}] is already being created.".format(
self.pool_id))
else:
logger.info("Create pool error: {}".format(err))
traceback.print_exc()
utils.print_batch_exception(err)
def create_and_submit_job(self, batch_client: batch.BatchExtensionsClient):
"""
Creates the Job that will be submitted to the batch service
:param batch_client: The batch client to use.
:type batch_client: `azure.batch.BatchExtensionsClient`
"""
logger.info('Creating Job [{}]... job will run on [{}]'.format(self.job_id, self.pool_id))
# load the template and parameters file
template = ctm.load_file(self.template_file)
parameters = ctm.load_file(self.parameters_file)
# overrides some of the parameters needed in the file, container SAS
# tokens need to be generated for the container
ctm.set_parameter_name(parameters, self.job_id)
ctm.set_parameter_storage_info(parameters, self.storage_info)
ctm.set_template_pool_id(parameters, self.pool_id)
# Submits the job
self.submit_job(batch_client, template, parameters)
def create_and_submit_job(self, batch_client: batch.BatchExtensionsClient):
"""
Creates the Job that will be submitted to the batch service
:param batch_client: The batch client to use.
:type batch_client: `azure.batch.BatchExtensionsClient`
"""
logger.info('Creating Job [{}]... job will run on [{}]'.format(
self.job_id, self.pool_id))
# load the template and parameters file
template = ctm.load_file(self.template_file)
parameters = ctm.load_file(self.parameters_file)
# overrides some of the parameters needed in the file, container SAS
# tokens need to be generated for the container
ctm.set_parameter_name(parameters, self.job_id)
ctm.set_parameter_storage_info(parameters, self.storage_info)
ctm.set_template_pool_id(parameters, self.pool_id)
# Submits the job
self.submit_job(batch_client, template, parameters)
def create_pool(self, batch_service_client: batch.BatchExtensionsClient,
image_references: 'List[utils.ImageReference]', VM_image_URL=None, VM_OS_type=None):
"""
Creates the Pool that will be submitted to the batch service.
:type batch_service_client: `azure.batch.BatchExtensionsClient`
:param batch_service_client: The batch client used for making batch operations
:type image_references: List['utils.ImageReference`]
:param image_references: A list of image references that job can run
:type VM_image_URL: str
:param VM_image_URL: The resource link to an image inside your image repo. If this is resource link is provided
the VMs will use the custom image you provided.
:type VM_OS_type: str
:param VM_OS_type: The custom image operating system type, this can be windows or centos. This is needed if you
want to use a custom image.
"""
# load the template file
template = ctm.load_file(self.pool_template_file)
# set extra license if needed
if self.application_licenses is not None:
template["pool"]["applicationLicenses"] = self.application_licenses.split(",")
# Set rendering version
ctm.set_image_reference(template, image_references)
ctm.set_template_pool_id(template, self.pool_id)
if VM_image_URL is not None and VM_OS_type is not None:
ctm.set_custom_image(template, VM_image_URL, VM_OS_type)
all_pools = [p.id for p in batch_service_client.pool.list()]
if self.pool_id not in all_pools:
self.submit_pool(batch_service_client, template)
else:
logger.info('pool [{}] already exists'.format(self.pool_id))