def submit_job_and_add_task(batch_client, job_id, vm_size, vm_count): """Submits a job to the Azure Batch service and adds a simple task with preparation task :param batch_client: The batch client to use. :type batch_client: `batchserviceclient.BatchServiceClient` :param str job_id: The id of the job to create. """ pool_info = batchmodels.PoolInformation( auto_pool_specification=batchmodels.AutoPoolSpecification( auto_pool_id_prefix="Helloworld_jobprep", pool=batchmodels.PoolSpecification( vm_size=vm_size, target_dedicated_nodes=vm_count, cloud_service_configuration={'os_family': "4"}), keep_alive=False, pool_lifetime_option=batchmodels.PoolLifetimeOption.job)) job = batchmodels.JobAddParameter( id=job_id, pool_info=pool_info, job_preparation_task=batch.models.JobPreparationTask( command_line=prep_task_command, wait_for_success=True)) batch_client.job.add(job) task = batchmodels.TaskAddParameter( id="HelloWorld_Task", command_line=common.helpers.wrap_commands_in_shell( 'windows', ['echo Hello world from the Batch Hello world sample!'])) batch_client.task.add(job_id=job.id, task=task)
def submit_job_and_add_task(batch_client, job_id, vm_size, vm_count): """Submits a job to the Azure Batch service and adds a simple task. :param batch_client: The batch client to use. :type batch_client: `batchserviceclient.BatchServiceClient` :param str job_id: The id of the job to create. """ pool_info = batchmodels.PoolInformation( auto_pool_specification=batchmodels.AutoPoolSpecification( auto_pool_id_prefix="HelloWorld", pool=batchmodels.PoolSpecification( vm_size=vm_size, target_dedicated=vm_count, cloud_service_configuration=batchmodels. CloudServiceConfiguration(os_family="4")), keep_alive=False, pool_lifetime_option=batchmodels.PoolLifetimeOption.job)) job = batchmodels.CloudJob(id=job_id, pool_info=pool_info) batch_client.job.add(job) task = batchmodels.CloudTask( id="HelloWorld", command_line=common.helpers.wrap_commands_in_shell( 'windows', ['echo Hello world from the Batch Hello world sample!'])) batch_client.task.add(job_id=job.id, task=task)
def test_batch_jobs(self, **kwargs): client = self.create_sharedkey_client(**kwargs) # Test Create Job auto_pool = models.AutoPoolSpecification( pool_lifetime_option=models.PoolLifetimeOption.job, pool=models.PoolSpecification( vm_size='small', cloud_service_configuration=models.CloudServiceConfiguration( os_family='5' ) ) ) job_prep = models.JobPreparationTask(command_line="cmd /c \"echo hello world\"") job_release = models.JobReleaseTask(command_line="cmd /c \"echo goodbye world\"") job_param = models.JobAddParameter( id=self.get_resource_name('batch_job1_'), pool_info=models.PoolInformation( auto_pool_specification=auto_pool ), job_preparation_task=job_prep, job_release_task=job_release ) response = client.job.add(job_param) self.assertIsNone(response) # Test Update Job constraints = models.JobConstraints(max_task_retry_count=3) options = models.JobUpdateParameter( priority=500, constraints=constraints, pool_info=models.PoolInformation( auto_pool_specification=auto_pool ) ) response = client.job.update(job_param.id, options) self.assertIsNone(response) # Test Patch Job options = models.JobPatchParameter(priority=900) response = client.job.patch(job_param.id, options) self.assertIsNone(response) job = client.job.get(job_param.id) self.assertIsInstance(job, models.CloudJob) self.assertEqual(job.id, job_param.id) self.assertEqual(job.constraints.max_task_retry_count, 3) self.assertEqual(job.priority, 900) # Test Create Job with Auto Complete job_auto_param = models.JobAddParameter( id=self.get_resource_name('batch_job2_'), on_all_tasks_complete=models.OnAllTasksComplete.terminate_job, on_task_failure=models.OnTaskFailure.perform_exit_options_job_action, pool_info=models.PoolInformation( auto_pool_specification=auto_pool ) ) response = client.job.add(job_auto_param) self.assertIsNone(response) job = client.job.get(job_auto_param.id) self.assertIsInstance(job, models.CloudJob) self.assertEqual(job.on_all_tasks_complete, models.OnAllTasksComplete.terminate_job) self.assertEqual(job.on_task_failure, models.OnTaskFailure.perform_exit_options_job_action) # Test List Jobs jobs = client.job.list() self.assertIsInstance(jobs, models.CloudJobPaged) self.assertEqual(len(list(jobs)), 2) # Test Disable Job response = client.job.disable(job_param.id, models.DisableJobOption.requeue) self.assertIsNone(response) # Test Enable Job response = client.job.enable(job_param.id) self.assertIsNone(response) # Prep and release task status task_status = client.job.list_preparation_and_release_task_status(job_param.id) self.assertIsInstance(task_status, models.JobPreparationAndReleaseTaskExecutionInformationPaged) self.assertEqual(list(task_status), []) # Test Terminate Job response = client.job.terminate(job_param.id) self.assertIsNone(response) # Test Delete Job response = client.job.delete(job_auto_param.id) self.assertIsNone(response) # Test Job Lifetime Statistics stats = client.job.get_all_lifetime_statistics() self.assertIsInstance(stats, models.JobStatistics) self.assertEqual(stats.num_succeeded_tasks, 0) self.assertEqual(stats.num_failed_tasks, 0)
def __submit_job(self, job_configuration, start_task, job_manager_task, autoscale_formula, software_metadata_key: str, vm_image_model, application_metadata): """ Job Submission :param job_configuration -> aztk_sdk.spark.models.JobConfiguration :param start_task -> batch_models.StartTask :param job_manager_task -> batch_models.TaskAddParameter :param autoscale forumula -> str :param software_metadata_key -> str :param vm_image_model -> aztk_sdk.models.VmImage :returns None """ self._get_cluster_data(job_configuration.id).save_cluster_config(job_configuration.to_cluster_config()) # get a verified node agent sku sku_to_use, image_ref_to_use = \ helpers.select_latest_verified_vm_image_with_node_agent_sku( vm_image_model.publisher, vm_image_model.offer, vm_image_model.sku, self.batch_client) # set up subnet if necessary network_conf = None if job_configuration.subnet_id: network_conf = batch_models.NetworkConfiguration( subnet_id=job_configuration.subnet_id) # set up a schedule for a recurring job auto_pool_specification = batch_models.AutoPoolSpecification( pool_lifetime_option=batch_models.PoolLifetimeOption.job_schedule, auto_pool_id_prefix=job_configuration.id, keep_alive=False, pool=batch_models.PoolSpecification( display_name=job_configuration.id, virtual_machine_configuration=batch_models.VirtualMachineConfiguration( image_reference=image_ref_to_use, node_agent_sku_id=sku_to_use), vm_size=job_configuration.vm_size, enable_auto_scale=True, auto_scale_formula=autoscale_formula, auto_scale_evaluation_interval=timedelta(minutes=5), start_task=start_task, enable_inter_node_communication=not job_configuration.mixed_mode(), network_configuration=network_conf, max_tasks_per_node=4, metadata=[ batch_models.MetadataItem( name=constants.AZTK_SOFTWARE_METADATA_KEY, value=software_metadata_key), batch_models.MetadataItem( name=constants.AZTK_MODE_METADATA_KEY, value=constants.AZTK_JOB_MODE_METADATA) ] ) ) # define job specification job_spec = batch_models.JobSpecification( pool_info=batch_models.PoolInformation(auto_pool_specification=auto_pool_specification), display_name=job_configuration.id, on_all_tasks_complete=batch_models.OnAllTasksComplete.terminate_job, job_manager_task=job_manager_task, metadata=[ batch_models.MetadataItem( name='applications', value=application_metadata) ] ) # define schedule schedule = batch_models.Schedule( do_not_run_until=None, do_not_run_after=None, start_window=None, recurrence_interval=None ) # create job schedule and add task setup = batch_models.JobScheduleAddParameter( id=job_configuration.id, schedule=schedule, job_specification=job_spec) self.batch_client.job_schedule.add(setup) return self.batch_client.job_schedule.get(job_schedule_id=job_configuration.id)
def create_job_schedule(batch_client, job_schedule_id, vm_size, vm_count, block_blob_client): """Creates an Azure Batch pool and job schedule with the specified ids. :param batch_client: The batch client to use. :type batch_client: `batchserviceclient.BatchServiceClient` :param str job_schedule_id: The id of the job schedule to create :param str vm_size: vm size (sku) :param int vm_count: number of vms to allocate :param block_blob_client: The storage block blob client to use. :type block_blob_client: `azure.storage.blob.BlockBlobService` """ cloud_service_config = batchmodels.CloudServiceConfiguration(os_family='6') user_id = batchmodels.UserIdentity( auto_user=batchmodels.AutoUserSpecification( elevation_level=_USER_ELEVATION_LEVEL)) python_download = batchmodels.ResourceFile(http_url=_PYTHON_DOWNLOAD, file_path='python373.exe') pool_info = batchmodels.PoolInformation( auto_pool_specification=batchmodels.AutoPoolSpecification( auto_pool_id_prefix="JobScheduler", pool=batchmodels.PoolSpecification( vm_size=vm_size, target_dedicated_nodes=vm_count, cloud_service_configuration=cloud_service_config, start_task=batchmodels.StartTask( command_line=common.helpers.wrap_commands_in_shell( 'windows', ['{}'.format(_PYTHON_INSTALL)]), resource_files=[python_download], wait_for_success=True, user_identity=user_id)), keep_alive=False, pool_lifetime_option=batchmodels.PoolLifetimeOption.job)) sas_url = common.helpers.upload_blob_and_create_sas( block_blob_client, _CONTAINER_NAME, _SIMPLE_TASK_NAME, _SIMPLE_TASK_PATH, datetime.datetime.utcnow() + datetime.timedelta(minutes=30)) job_spec = batchmodels.JobSpecification( pool_info=pool_info, # Terminate job once all tasks under it are complete to allow for a new # job to be created under the schedule on_all_tasks_complete=batchmodels.OnAllTasksComplete.terminate_job, job_manager_task=batchmodels.JobManagerTask( id="JobManagerTask", command_line=common.helpers.wrap_commands_in_shell( 'windows', ['python {}'.format(_SIMPLE_TASK_NAME)]), resource_files=[ batchmodels.ResourceFile(file_path=_SIMPLE_TASK_NAME, http_url=sas_url) ])) do_not_run_after = datetime.datetime.utcnow() \ + datetime.timedelta(minutes=30) schedule = batchmodels.Schedule( do_not_run_after=do_not_run_after, recurrence_interval=datetime.timedelta(minutes=10)) scheduled_job = batchmodels.JobScheduleAddParameter( id=job_schedule_id, schedule=schedule, job_specification=job_spec) batch_client.job_schedule.add(cloud_job_schedule=scheduled_job)
def create_job_schedule(batch_client, job_schedule_id, vm_size, vm_count): """Creates an Azure Batch pool and job schedule with the specified ids. :param batch_client: The batch client to use. :type batch_client: `batchserviceclient.BatchServiceClient` :param str job_schedule_id: The id of the job schedule to create :param str vm_size: vm size (sku) :param int vm_count: number of vms to allocate """ pool_info = batchmodels.PoolInformation( auto_pool_specification=batchmodels.AutoPoolSpecification( auto_pool_id_prefix="JobScheduler", pool=batchmodels.PoolSpecification( vm_size=vm_size, target_dedicated_nodes=vm_count, virtual_machine_configuration=batchmodels.VirtualMachineConfiguration( image_reference=batchmodels.ImageReference( publisher="Canonical", offer="UbuntuServer", sku="18.04-LTS", version="latest" ), node_agent_sku_id="batch.node.ubuntu 18.04" ), start_task=batchmodels.StartTask( command_line="/bin/bash -c " "\"$AZ_BATCH_APP_PACKAGE_azure_batch_1/azure_batch/job_schedular_node_startup_tasks.sh\"", wait_for_success=True, user_identity=batchmodels.UserIdentity( auto_user=batchmodels.AutoUserSpecification( scope=batchmodels.AutoUserScope.pool, elevation_level=batchmodels.ElevationLevel.admin) ), ), application_package_references=[batchmodels.ApplicationPackageReference( application_id="azure_batch", version="1" )], ), keep_alive=False, pool_lifetime_option=batchmodels.PoolLifetimeOption.job ) ) job_spec = batchmodels.JobSpecification( pool_info=pool_info, # Terminate job once all tasks under it are complete to allow for a new # job to be created under the schedule on_all_tasks_complete=batchmodels.OnAllTasksComplete.terminate_job, job_manager_task=batchmodels.JobManagerTask( id="JobManagerTask", #specify the command that needs to run recursively in job_schedular command_line="/bin/bash -c \" python3 " "$AZ_BATCH_APP_PACKAGE_azure_batch_1/azure_batch/azure_batch_main.py\"" )) #mention the interval of the job schedular schedule = batchmodels.Schedule( recurrence_interval=datetime.timedelta(days=15)) scheduled_job = batchmodels.JobScheduleAddParameter( id=job_schedule_id, schedule=schedule, job_specification=job_spec) batch_client.job_schedule.add(cloud_job_schedule=scheduled_job)