def generate_task(spark_client, job, application_tasks): resource_files = [] for application, task in application_tasks: task_definition_resource_file = helpers.upload_text_to_container( container_name=job.id, application_name=application.name + ".yaml", file_path=application.name + ".yaml", content=yaml.dump(task), blob_client=spark_client.blob_client, ) resource_files.append(task_definition_resource_file) task_cmd = __app_cmd() # Create task task = batch_models.JobManagerTask( id=job.id, command_line=helpers.wrap_commands_in_shell([task_cmd]), resource_files=resource_files, kill_job_on_completion=False, allow_low_priority_node=True, user_identity=batch_models.UserIdentity( auto_user=batch_models.AutoUserSpecification( scope=batch_models.AutoUserScope.task, elevation_level=batch_models.ElevationLevel.admin)), ) return task
def create_job_schedule(batch_client, job_schedule_id, vm_size, vm_count, block_blob_client): """Creates an Azure Batch pool and job schedule with the specified ids. :param batch_client: The batch client to use. :type batch_client: `batchserviceclient.BatchServiceClient` :param str job_schedule_id: The id of the job schedule to create :param str vm_size: vm size (sku) :param int vm_count: number of vms to allocate :param block_blob_client: The storage block blob client to use. :type block_blob_client: `azure.storage.blob.BlockBlobService` """ cloud_service_config = batchmodels.CloudServiceConfiguration(os_family='6') user_id = batchmodels.UserIdentity( auto_user=batchmodels.AutoUserSpecification( elevation_level=_USER_ELEVATION_LEVEL)) python_download = batchmodels.ResourceFile(http_url=_PYTHON_DOWNLOAD, file_path='python373.exe') pool_info = batchmodels.PoolInformation( auto_pool_specification=batchmodels.AutoPoolSpecification( auto_pool_id_prefix="JobScheduler", pool=batchmodels.PoolSpecification( vm_size=vm_size, target_dedicated_nodes=vm_count, cloud_service_configuration=cloud_service_config, start_task=batchmodels.StartTask( command_line=common.helpers.wrap_commands_in_shell( 'windows', ['{}'.format(_PYTHON_INSTALL)]), resource_files=[python_download], wait_for_success=True, user_identity=user_id)), keep_alive=False, pool_lifetime_option=batchmodels.PoolLifetimeOption.job)) sas_url = common.helpers.upload_blob_and_create_sas( block_blob_client, _CONTAINER_NAME, _SIMPLE_TASK_NAME, _SIMPLE_TASK_PATH, datetime.datetime.utcnow() + datetime.timedelta(minutes=30)) job_spec = batchmodels.JobSpecification( pool_info=pool_info, # Terminate job once all tasks under it are complete to allow for a new # job to be created under the schedule on_all_tasks_complete=batchmodels.OnAllTasksComplete.terminate_job, job_manager_task=batchmodels.JobManagerTask( id="JobManagerTask", command_line=common.helpers.wrap_commands_in_shell( 'windows', ['python {}'.format(_SIMPLE_TASK_NAME)]), resource_files=[ batchmodels.ResourceFile(file_path=_SIMPLE_TASK_NAME, http_url=sas_url) ])) do_not_run_after = datetime.datetime.utcnow() \ + datetime.timedelta(minutes=30) schedule = batchmodels.Schedule( do_not_run_after=do_not_run_after, recurrence_interval=datetime.timedelta(minutes=10)) scheduled_job = batchmodels.JobScheduleAddParameter( id=job_schedule_id, schedule=schedule, job_specification=job_spec) batch_client.job_schedule.add(cloud_job_schedule=scheduled_job)
def create_job_schedule(batch_client, job_schedule_id, vm_size, vm_count): """Creates an Azure Batch pool and job schedule with the specified ids. :param batch_client: The batch client to use. :type batch_client: `batchserviceclient.BatchServiceClient` :param str job_schedule_id: The id of the job schedule to create :param str vm_size: vm size (sku) :param int vm_count: number of vms to allocate """ pool_info = batchmodels.PoolInformation( auto_pool_specification=batchmodels.AutoPoolSpecification( auto_pool_id_prefix="JobScheduler", pool=batchmodels.PoolSpecification( vm_size=vm_size, target_dedicated_nodes=vm_count, virtual_machine_configuration=batchmodels.VirtualMachineConfiguration( image_reference=batchmodels.ImageReference( publisher="Canonical", offer="UbuntuServer", sku="18.04-LTS", version="latest" ), node_agent_sku_id="batch.node.ubuntu 18.04" ), start_task=batchmodels.StartTask( command_line="/bin/bash -c " "\"$AZ_BATCH_APP_PACKAGE_azure_batch_1/azure_batch/job_schedular_node_startup_tasks.sh\"", wait_for_success=True, user_identity=batchmodels.UserIdentity( auto_user=batchmodels.AutoUserSpecification( scope=batchmodels.AutoUserScope.pool, elevation_level=batchmodels.ElevationLevel.admin) ), ), application_package_references=[batchmodels.ApplicationPackageReference( application_id="azure_batch", version="1" )], ), keep_alive=False, pool_lifetime_option=batchmodels.PoolLifetimeOption.job ) ) job_spec = batchmodels.JobSpecification( pool_info=pool_info, # Terminate job once all tasks under it are complete to allow for a new # job to be created under the schedule on_all_tasks_complete=batchmodels.OnAllTasksComplete.terminate_job, job_manager_task=batchmodels.JobManagerTask( id="JobManagerTask", #specify the command that needs to run recursively in job_schedular command_line="/bin/bash -c \" python3 " "$AZ_BATCH_APP_PACKAGE_azure_batch_1/azure_batch/azure_batch_main.py\"" )) #mention the interval of the job schedular schedule = batchmodels.Schedule( recurrence_interval=datetime.timedelta(days=15)) scheduled_job = batchmodels.JobScheduleAddParameter( id=job_schedule_id, schedule=schedule, job_specification=job_spec) batch_client.job_schedule.add(cloud_job_schedule=scheduled_job)