def __create_pool_and_job(self, cluster_conf: models.ClusterConfiguration, software_metadata_key: str, start_task, VmImageModel): """ Create a pool and job :param cluster_conf: the configuration object used to create the cluster :type cluster_conf: aztk.models.ClusterConfiguration :parm software_metadata_key: the id of the software being used on the cluster :param start_task: the start task for the cluster :param VmImageModel: the type of image to provision for the cluster :param wait: wait until the cluster is ready """ self._get_cluster_data(cluster_conf.cluster_id).save_cluster_config(cluster_conf) # reuse pool_id as job_id pool_id = cluster_conf.cluster_id job_id = cluster_conf.cluster_id # Get a verified node agent sku sku_to_use, image_ref_to_use = \ helpers.select_latest_verified_vm_image_with_node_agent_sku( VmImageModel.publisher, VmImageModel.offer, VmImageModel.sku, self.batch_client) network_conf = None if cluster_conf.subnet_id is not None: network_conf = batch_models.NetworkConfiguration( subnet_id=cluster_conf.subnet_id) auto_scale_formula = "$TargetDedicatedNodes={0}; $TargetLowPriorityNodes={1}".format( cluster_conf.vm_count, cluster_conf.vm_low_pri_count) # Confiure the pool pool = batch_models.PoolAddParameter( id=pool_id, virtual_machine_configuration=batch_models.VirtualMachineConfiguration( image_reference=image_ref_to_use, node_agent_sku_id=sku_to_use), vm_size=cluster_conf.vm_size, enable_auto_scale=True, auto_scale_formula=auto_scale_formula, auto_scale_evaluation_interval=timedelta(minutes=5), start_task=start_task, enable_inter_node_communication=True if not cluster_conf.subnet_id else False, max_tasks_per_node=4, network_configuration=network_conf, metadata=[ batch_models.MetadataItem( name=constants.AZTK_SOFTWARE_METADATA_KEY, value=software_metadata_key), batch_models.MetadataItem( name=constants.AZTK_MODE_METADATA_KEY, value=constants.AZTK_CLUSTER_MODE_METADATA) ]) # Create the pool + create user for the pool helpers.create_pool_if_not_exist(pool, self.batch_client) # Create job job = batch_models.JobAddParameter( id=job_id, pool_info=batch_models.PoolInformation(pool_id=pool_id)) # Add job to batch self.batch_client.job.add(job) return helpers.get_cluster(cluster_conf.cluster_id, self.batch_client)
def create_job(batch_service_client, job_id, pool_id): """ Creates a job with the specified ID, associated with the specified pool. :param batch_service_client: A Batch service client. :type batch_service_client: `azure.batch.BatchServiceClient` :param str job_id: The ID for the job. :param str pool_id: The ID for the pool. """ print('Creating job [{}]...'.format(job_id)) job = batch.models.JobAddParameter( job_id, batch.models.PoolInformation(pool_id=pool_id), metadata=[ batchmodels.MetadataItem("AZURE_TENANT_ID", _AZURE_TENANT_ID), batchmodels.MetadataItem("AZURE_SUBSCRIPTION_ID", _AZURE_SUBSCRIPTION_ID), batchmodels.MetadataItem("AZURE_CLIENT_ID", _AZURE_CLIENT_ID), batchmodels.MetadataItem("AZURE_CLIENT_OID", _AZURE_CLIENT_OID), batchmodels.MetadataItem("AZURE_CLIENT_SECRET", _AZURE_CLIENT_SECRET) ]) batch_service_client.job.add(job)
def __create_pool_and_job(self, cluster_conf, software_metadata_key: str, start_task, VmImageModel): """ Create a pool and job :param cluster_conf: the configuration object used to create the cluster :type cluster_conf: aztk.models.ClusterConfiguration :parm software_metadata_key: the id of the software being used on the cluster :param start_task: the start task for the cluster :param VmImageModel: the type of image to provision for the cluster :param wait: wait until the cluster is ready """ # reuse pool_id as job_id pool_id = cluster_conf.cluster_id job_id = cluster_conf.cluster_id # Get a verified node agent sku sku_to_use, image_ref_to_use = \ helpers.select_latest_verified_vm_image_with_node_agent_sku( VmImageModel.publisher, VmImageModel.offer, VmImageModel.sku, self.batch_client) # Confiure the pool pool = batch_models.PoolAddParameter( id=pool_id, virtual_machine_configuration=batch_models. VirtualMachineConfiguration(image_reference=image_ref_to_use, node_agent_sku_id=sku_to_use), vm_size=cluster_conf.vm_size, target_dedicated_nodes=cluster_conf.vm_count, target_low_priority_nodes=cluster_conf.vm_low_pri_count, start_task=start_task, enable_inter_node_communication=True, max_tasks_per_node=1, metadata=[ batch_models.MetadataItem( name=constants.AZTK_SOFTWARE_METADATA_KEY, value=software_metadata_key), ]) # Create the pool + create user for the pool helpers.create_pool_if_not_exist(pool, self.batch_client) # Create job job = batch_models.JobAddParameter( id=job_id, pool_info=batch_models.PoolInformation(pool_id=pool_id)) # Add job to batch self.batch_client.job.add(job) return helpers.get_cluster(cluster_conf.cluster_id, self.batch_client)
def test_batch_update_pools(self, **kwargs): client = self.create_sharedkey_client(**kwargs) # Test Create Paas Pool test_paas_pool = models.PoolAddParameter( id=self.get_resource_name('batch_paas_'), vm_size='small', cloud_service_configuration=models.CloudServiceConfiguration( os_family='5' ), start_task=models.StartTask( command_line="cmd.exe /c \"echo hello world\"", resource_files=[models.ResourceFile('https://blobsource.com', 'filename.txt')], environment_settings=[models.EnvironmentSetting('ENV_VAR', 'env_value')], user_identity=models.UserIdentity( auto_user=models.AutoUserSpecification( elevation_level=models.ElevationLevel.admin ) ) ) ) response = client.pool.add(test_paas_pool) self.assertIsNone(response) # Test Upgrade Pool OS self.assertBatchError( "PoolVersionEqualsUpgradeVersion", client.pool.upgrade_os, test_paas_pool.id, "*" ) # Test Update Pool Parameters params = models.PoolUpdatePropertiesParameter([], [], [models.MetadataItem('foo', 'bar')]) response = client.pool.update_properties(test_paas_pool.id, params) self.assertIsNone(response) # Test Patch Pool Parameters params = models.PoolPatchParameter(metadata=[models.MetadataItem('foo2', 'bar2')]) response = client.pool.patch(test_paas_pool.id, params) self.assertIsNone(response) # Test Pool Exists response = client.pool.exists(test_paas_pool.id) self.assertTrue(response) # Test Get Pool pool = client.pool.get(test_paas_pool.id) self.assertIsInstance(pool, models.CloudPool) self.assertEqual(pool.id, test_paas_pool.id) self.assertEqual(pool.state, models.PoolState.active) self.assertEqual(pool.allocation_state, models.AllocationState.steady) self.assertEqual(pool.cloud_service_configuration.os_family, '5') self.assertEqual(pool.vm_size, 'small') self.assertIsNone(pool.start_task) self.assertEqual(pool.metadata[0].name, 'foo2') self.assertEqual(pool.metadata[0].value, 'bar2') # Test Get Pool with OData Clauses options = models.PoolGetOptions(select='id,state', expand='stats') pool = client.pool.get(test_paas_pool.id, options) self.assertIsInstance(pool, models.CloudPool) self.assertEqual(pool.id, test_paas_pool.id) self.assertEqual(pool.state, models.PoolState.active) self.assertIsNone(pool.allocation_state) self.assertIsNone(pool.vm_size) # Test Delete Pool response = client.pool.delete(test_paas_pool.id) self.assertIsNone(response)
def __submit_job(self, job_configuration, start_task, job_manager_task, autoscale_formula, software_metadata_key: str, vm_image_model, application_metadata): """ Job Submission :param job_configuration -> aztk_sdk.spark.models.JobConfiguration :param start_task -> batch_models.StartTask :param job_manager_task -> batch_models.TaskAddParameter :param autoscale forumula -> str :param software_metadata_key -> str :param vm_image_model -> aztk_sdk.models.VmImage :returns None """ self._get_cluster_data(job_configuration.id).save_cluster_config(job_configuration.to_cluster_config()) # get a verified node agent sku sku_to_use, image_ref_to_use = \ helpers.select_latest_verified_vm_image_with_node_agent_sku( vm_image_model.publisher, vm_image_model.offer, vm_image_model.sku, self.batch_client) # set up subnet if necessary network_conf = None if job_configuration.subnet_id: network_conf = batch_models.NetworkConfiguration( subnet_id=job_configuration.subnet_id) # set up a schedule for a recurring job auto_pool_specification = batch_models.AutoPoolSpecification( pool_lifetime_option=batch_models.PoolLifetimeOption.job_schedule, auto_pool_id_prefix=job_configuration.id, keep_alive=False, pool=batch_models.PoolSpecification( display_name=job_configuration.id, virtual_machine_configuration=batch_models.VirtualMachineConfiguration( image_reference=image_ref_to_use, node_agent_sku_id=sku_to_use), vm_size=job_configuration.vm_size, enable_auto_scale=True, auto_scale_formula=autoscale_formula, auto_scale_evaluation_interval=timedelta(minutes=5), start_task=start_task, enable_inter_node_communication=not job_configuration.mixed_mode(), network_configuration=network_conf, max_tasks_per_node=4, metadata=[ batch_models.MetadataItem( name=constants.AZTK_SOFTWARE_METADATA_KEY, value=software_metadata_key), batch_models.MetadataItem( name=constants.AZTK_MODE_METADATA_KEY, value=constants.AZTK_JOB_MODE_METADATA) ] ) ) # define job specification job_spec = batch_models.JobSpecification( pool_info=batch_models.PoolInformation(auto_pool_specification=auto_pool_specification), display_name=job_configuration.id, on_all_tasks_complete=batch_models.OnAllTasksComplete.terminate_job, job_manager_task=job_manager_task, metadata=[ batch_models.MetadataItem( name='applications', value=application_metadata) ] ) # define schedule schedule = batch_models.Schedule( do_not_run_until=None, do_not_run_after=None, start_window=None, recurrence_interval=None ) # create job schedule and add task setup = batch_models.JobScheduleAddParameter( id=job_configuration.id, schedule=schedule, job_specification=job_spec) self.batch_client.job_schedule.add(setup) return self.batch_client.job_schedule.get(job_schedule_id=job_configuration.id)