def create_pool(self, batch_service_client: batch.BatchExtensionsClient, image_references: 'List[utils.ImageReference]'): """ Creates the Pool that will be submitted to the batch service. :type batch_service_client: `azure.batch.BatchExtensionsClient` :param batch_service_client: The batch client used for making batch operations :type image_references: List['utils.ImageReference`] :param image_references: A list of image references that job can run """ # load the template file template = ctm.load_file(self.pool_template_file) # set extra license if needed if self.application_licenses is not None: template["pool"][ "applicationLicenses"] = self.application_licenses.split(",") # Set rendering version ctm.set_image_reference(template, image_references) ctm.set_template_pool_id(template, self.pool_id) all_pools = [p.id for p in batch_service_client.pool.list()] if self.pool_id not in all_pools: self.submit_pool(batch_service_client, template) else: logger.info('pool [{}] already exists'.format(self.pool_id))
def submit_pool(self, batch_service_client: batch.BatchExtensionsClient, template: str): """ Submits a batch pool based on the template :param batch_service_client: The batch client used for making batch operations :type batch_service_client: `azure.batch.BatchExtensionsClient` :param template: The in memory version of the template used to create a the job. :type template: str """ parameters = ctm.load_file(self.parameters_file) #updates any placeholder parameter values with the values from keyVault, if required utils.update_params_with_values_from_keyvault( parameters, self.keyvault_client_with_url) pool_json = batch_service_client.pool.expand_template( template, parameters) ctm.set_template_pool_id(template, self.pool_id) pool = batch_service_client.pool.poolparameter_from_json(pool_json) logger.info('Creating pool [{}]...'.format(pool)) try: batch_service_client.pool.add(pool) except batchmodels.BatchErrorException as err: if utils.expected_exception(err, "The specified pool already exists"): logger.warning("Pool [{}] is already being created.".format( self.pool_id)) else: logger.info("Create pool error: {}".format(err)) traceback.print_exc() utils.print_batch_exception(err)
def create_and_submit_job(self, batch_client: batch.BatchExtensionsClient): """ Creates the Job that will be submitted to the batch service :param batch_client: The batch client to use. :type batch_client: `azure.batch.BatchExtensionsClient` """ logger.info('Creating Job [{}]... job will run on [{}]'.format( self.job_id, self.pool_id)) # load the template and parameters file template = ctm.load_file(self.template_file) parameters = ctm.load_file(self.parameters_file) #updates any placeholder parameter values with the values from keyVault, if required utils.update_params_with_values_from_keyvault( parameters, self.keyvault_client_with_url) # overrides some of the parameters needed in the file, container SAS # tokens need to be generated for the container ctm.set_parameter_name(parameters, self.job_id) ctm.set_parameter_storage_info(parameters, self.storage_info) ctm.set_template_pool_id(parameters, self.pool_id) ctm.set_job_resource_file_urls_to_branch(template, self.repository_branch_name) # Submits the job self.submit_job(batch_client, template, parameters)
def submit_pool(self, batch_service_client: batch.BatchExtensionsClient, template: str): """ Submits a batch pool based on the template :param batch_service_client: The batch client used for making batch operations :type batch_service_client: `azure.batch.BatchExtensionsClient` :param template: The in memory version of the template used to create a the job. :type template: str """ parameters = ctm.load_file(self.parameters_file) pool_json = batch_service_client.pool.expand_template(template, parameters) ctm.set_template_pool_id(template, self.pool_id) pool = batch_service_client.pool.poolparameter_from_json(pool_json) logger.info('Creating pool [{}]...'.format(self.pool_id)) try: batch_service_client.pool.add(pool) except batchmodels.batch_error.BatchErrorException as err: if utils.expected_exception( err, "The specified pool already exists"): logger.warning( "Pool [{}] is already being created.".format( self.pool_id)) else: logger.info("Create pool error: {}".format(err)) traceback.print_exc() utils.print_batch_exception(err)
def create_pool(self, batch_service_client: batch.BatchExtensionsClient, image_references: 'List[utils.ImageReference]', VM_image_URL=None, VM_OS_type=None): """ Creates the Pool that will be submitted to the batch service. :type batch_service_client: `azure.batch.BatchExtensionsClient` :param batch_service_client: The batch client used for making batch operations :type image_references: List['utils.ImageReference`] :param image_references: A list of image references that job can run :type VM_image_URL: str :param VM_image_URL: The resource link to an image inside your image repo. If this is resource link is provided the VMs will use the custom image you provided. :type VM_OS_type: str :param VM_OS_type: The custom image operating system type, this can be windows or centos. This is needed if you want to use a custom image. """ # load the template file template = ctm.load_file(self.pool_template_file) # set extra license if needed if self.application_licenses is not None: template["pool"][ "applicationLicenses"] = self.application_licenses.split(",") # Set rendering version ctm.set_image_reference(template, image_references) ctm.set_template_pool_id(template, self.pool_id) ctm.set_pool_resource_file_urls_to_branch(template, self.repository_branch_name) if VM_image_URL is not None: ctm.set_custom_image(template, VM_image_URL, VM_OS_type) all_pools = [p.id for p in batch_service_client.pool.list()] if self.pool_id not in all_pools: self.submit_pool(batch_service_client, template) else: logger.info('pool [{}] already exists'.format(self.pool_id))
def submit_pool(self, batch_service_client: batch.BatchExtensionsClient, template: str, UseLowPriorityVMs: bool): """ Submits a batch pool based on the template :param batch_service_client: The batch client used for making batch operations :type batch_service_client: `azure.batch.BatchExtensionsClient` :param template: The in memory version of the template used to create a the job. :type template: str :param UseLowPriorityVMs: True: Use low priority nodes, False: will use the type in the template file :type UseLowPriorityVMs: bool """ parameters = ctm.load_file(self.parameters_file) if UseLowPriorityVMs==True: #swap the dedicated vm count to low pri and zero out the dedicated count (to reduce testing COGS) ctm.set_low_priority_vm_count(template, str(self.min_required_vms)) #some tests set the dedicated vm count in the test parameters file, and some use the default in the template, so override both to 0 ctm.set_dedicated_vm_count(parameters, 0) ctm.set_dedicated_vm_count(template, 0) # updates any placeholder parameter values with the values from # keyVault, if required utils.update_params_with_values_from_keyvault( parameters, self.keyvault_client_with_url) pool_json = batch_service_client.pool.expand_template( template, parameters) ctm.set_template_pool_id(template, self.pool_id) pool = batch_service_client.pool.poolparameter_from_json(pool_json) logger.info('Creating pool [{}]...'.format(pool)) try: utils.run_with_jitter_retry(batch_service_client.pool.add, pool) except batchmodels.BatchErrorException as err: if utils.expected_exception( err, "The specified pool already exists"): logger.warning( "Pool [{}] is already being created.".format( self.pool_id)) else: logger.info("Create pool error: {}".format(err)) traceback.print_exc() utils.print_batch_exception(err)
def create_and_submit_job(self, batch_client: batch.BatchExtensionsClient): """ Creates the Job that will be submitted to the batch service :param batch_client: The batch client to use. :type batch_client: `azure.batch.BatchExtensionsClient` """ logger.info('Creating Job [{}]... job will run on [{}]'.format(self.job_id, self.pool_id)) # load the template and parameters file template = ctm.load_file(self.template_file) parameters = ctm.load_file(self.parameters_file) # overrides some of the parameters needed in the file, container SAS # tokens need to be generated for the container ctm.set_parameter_name(parameters, self.job_id) ctm.set_parameter_storage_info(parameters, self.storage_info) ctm.set_template_pool_id(parameters, self.pool_id) # Submits the job self.submit_job(batch_client, template, parameters)
def create_and_submit_job(self, batch_client: batch.BatchExtensionsClient): """ Creates the Job that will be submitted to the batch service :param batch_client: The batch client to use. :type batch_client: `azure.batch.BatchExtensionsClient` """ logger.info('Creating Job [{}]... job will run on [{}]'.format( self.job_id, self.pool_id)) # load the template and parameters file template = ctm.load_file(self.template_file) parameters = ctm.load_file(self.parameters_file) # overrides some of the parameters needed in the file, container SAS # tokens need to be generated for the container ctm.set_parameter_name(parameters, self.job_id) ctm.set_parameter_storage_info(parameters, self.storage_info) ctm.set_template_pool_id(parameters, self.pool_id) # Submits the job self.submit_job(batch_client, template, parameters)
def create_pool(self, batch_service_client: batch.BatchExtensionsClient, image_references: 'List[utils.ImageReference]', VM_image_URL=None, VM_OS_type=None): """ Creates the Pool that will be submitted to the batch service. :type batch_service_client: `azure.batch.BatchExtensionsClient` :param batch_service_client: The batch client used for making batch operations :type image_references: List['utils.ImageReference`] :param image_references: A list of image references that job can run :type VM_image_URL: str :param VM_image_URL: The resource link to an image inside your image repo. If this is resource link is provided the VMs will use the custom image you provided. :type VM_OS_type: str :param VM_OS_type: The custom image operating system type, this can be windows or centos. This is needed if you want to use a custom image. """ # load the template file template = ctm.load_file(self.pool_template_file) # set extra license if needed if self.application_licenses is not None: template["pool"]["applicationLicenses"] = self.application_licenses.split(",") # Set rendering version ctm.set_image_reference(template, image_references) ctm.set_template_pool_id(template, self.pool_id) if VM_image_URL is not None and VM_OS_type is not None: ctm.set_custom_image(template, VM_image_URL, VM_OS_type) all_pools = [p.id for p in batch_service_client.pool.list()] if self.pool_id not in all_pools: self.submit_pool(batch_service_client, template) else: logger.info('pool [{}] already exists'.format(self.pool_id))