def newBatchJobSchedule(account, key, URL, job_id, pool_id):
creds = batchAuth.SharedKeyCredentials(account_name=account, key=key)
client_creds = clientAuth.BatchServiceClient(creds, URL)
stop_running = input('How many days do you have this job to run for?: ')
hours = input('How many hours would you like the recurrence interval to be for your schedule?: ')
time_to_run = datetime.datetime.utcnow() + datetime.timedelta(days=int(stop_running))
try:
pool = batchmodels.PoolInformation(pool_id=pool_id)
jobSpec = batchmodels.JobSpecification(pool_info=pool)
schedule = batchmodels.Schedule(do_not_run_after=time_to_run, recurrence_interval=datetime.timedelta(hours=int(hours)))
job = batchmodels.JobScheduleAddParameter(id=job_id, schedule=schedule, job_specification=jobSpec)
client_creds.job_schedule.add(cloud_job_schedule=job)
except Exception as e:
logging.error(msg=e)
def test_batch_job_schedules(self, **kwargs):
client = self.create_aad_client(**kwargs)
# Test Create Job Schedule
schedule_id = self.get_resource_name('batch_schedule_')
job_spec = models.JobSpecification(
pool_info=models.PoolInformation("pool_id"),
constraints=models.JobConstraints(max_task_retry_count=2),
on_all_tasks_complete=models.OnAllTasksComplete.terminate_job
)
schedule = models.Schedule(
start_window=datetime.timedelta(hours=1),
recurrence_interval=datetime.timedelta(days=1)
)
params = models.JobScheduleAddParameter(
schedule_id,
schedule,
job_spec
)
response = client.job_schedule.add(params)
self.assertIsNone(response)
# Test List Job Schedules
schedules = list(client.job_schedule.list())
self.assertTrue(len(schedules) > 0)
# Test Get Job Schedule
schedule = client.job_schedule.get(schedule_id)
self.assertIsInstance(schedule, models.CloudJobSchedule)
self.assertEqual(schedule.id, schedule_id)
self.assertEqual(schedule.state, models.JobScheduleState.active)
# Test Job Schedule Exists
exists = client.job_schedule.exists(schedule_id)
self.assertTrue(exists)
# Test List Jobs from Schedule
jobs = list(client.job.list_from_job_schedule(schedule_id))
self.assertTrue(len(jobs) > 0)
# Test Disable Job Schedule
response = client.job_schedule.disable(schedule_id)
self.assertIsNone(response)
# Test Enable Job Schedule
response = client.job_schedule.enable(schedule_id)
self.assertIsNone(response)
# Test Update Job Schedule
job_spec = models.JobSpecification(
pool_info=models.PoolInformation('pool_id')
)
schedule = models.Schedule(
recurrence_interval=datetime.timedelta(hours=10)
)
params = models.JobScheduleUpdateParameter(schedule, job_spec)
response = client.job_schedule.update(schedule_id, params)
self.assertIsNone(response)
# Test Patch Job Schedule
schedule = models.Schedule(
recurrence_interval=datetime.timedelta(hours=5)
)
params = models.JobSchedulePatchParameter(schedule)
response = client.job_schedule.patch(schedule_id, params)
self.assertIsNone(response)
# Test Terminate Job Schedule
response = client.job_schedule.terminate(schedule_id)
self.assertIsNone(response)
# Test Delete Job Schedule
response = client.job_schedule.delete(schedule_id)
self.assertIsNone(response)
def __submit_job(self,
job_configuration,
start_task,
job_manager_task,
autoscale_formula,
software_metadata_key: str,
vm_image_model,
application_metadata):
"""
Job Submission
:param job_configuration -> aztk_sdk.spark.models.JobConfiguration
:param start_task -> batch_models.StartTask
:param job_manager_task -> batch_models.TaskAddParameter
:param autoscale forumula -> str
:param software_metadata_key -> str
:param vm_image_model -> aztk_sdk.models.VmImage
:returns None
"""
self._get_cluster_data(job_configuration.id).save_cluster_config(job_configuration.to_cluster_config())
# get a verified node agent sku
sku_to_use, image_ref_to_use = \
helpers.select_latest_verified_vm_image_with_node_agent_sku(
vm_image_model.publisher, vm_image_model.offer, vm_image_model.sku, self.batch_client)
# set up subnet if necessary
network_conf = None
if job_configuration.subnet_id:
network_conf = batch_models.NetworkConfiguration(
subnet_id=job_configuration.subnet_id)
# set up a schedule for a recurring job
auto_pool_specification = batch_models.AutoPoolSpecification(
pool_lifetime_option=batch_models.PoolLifetimeOption.job_schedule,
auto_pool_id_prefix=job_configuration.id,
keep_alive=False,
pool=batch_models.PoolSpecification(
display_name=job_configuration.id,
virtual_machine_configuration=batch_models.VirtualMachineConfiguration(
image_reference=image_ref_to_use,
node_agent_sku_id=sku_to_use),
vm_size=job_configuration.vm_size,
enable_auto_scale=True,
auto_scale_formula=autoscale_formula,
auto_scale_evaluation_interval=timedelta(minutes=5),
start_task=start_task,
enable_inter_node_communication=not job_configuration.mixed_mode(),
network_configuration=network_conf,
max_tasks_per_node=4,
metadata=[
batch_models.MetadataItem(
name=constants.AZTK_SOFTWARE_METADATA_KEY, value=software_metadata_key),
batch_models.MetadataItem(
name=constants.AZTK_MODE_METADATA_KEY, value=constants.AZTK_JOB_MODE_METADATA)
]
)
)
# define job specification
job_spec = batch_models.JobSpecification(
pool_info=batch_models.PoolInformation(auto_pool_specification=auto_pool_specification),
display_name=job_configuration.id,
on_all_tasks_complete=batch_models.OnAllTasksComplete.terminate_job,
job_manager_task=job_manager_task,
metadata=[
batch_models.MetadataItem(
name='applications', value=application_metadata)
]
)
# define schedule
schedule = batch_models.Schedule(
do_not_run_until=None,
do_not_run_after=None,
start_window=None,
recurrence_interval=None
)
# create job schedule and add task
setup = batch_models.JobScheduleAddParameter(
id=job_configuration.id,
schedule=schedule,
job_specification=job_spec)
self.batch_client.job_schedule.add(setup)
return self.batch_client.job_schedule.get(job_schedule_id=job_configuration.id)
def create_job_schedule(batch_client, job_schedule_id, vm_size, vm_count,
block_blob_client):
"""Creates an Azure Batch pool and job schedule with the specified ids.
:param batch_client: The batch client to use.
:type batch_client: `batchserviceclient.BatchServiceClient`
:param str job_schedule_id: The id of the job schedule to create
:param str vm_size: vm size (sku)
:param int vm_count: number of vms to allocate
:param block_blob_client: The storage block blob client to use.
:type block_blob_client: `azure.storage.blob.BlockBlobService`
"""
cloud_service_config = batchmodels.CloudServiceConfiguration(os_family='6')
user_id = batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
elevation_level=_USER_ELEVATION_LEVEL))
python_download = batchmodels.ResourceFile(http_url=_PYTHON_DOWNLOAD,
file_path='python373.exe')
pool_info = batchmodels.PoolInformation(
auto_pool_specification=batchmodels.AutoPoolSpecification(
auto_pool_id_prefix="JobScheduler",
pool=batchmodels.PoolSpecification(
vm_size=vm_size,
target_dedicated_nodes=vm_count,
cloud_service_configuration=cloud_service_config,
start_task=batchmodels.StartTask(
command_line=common.helpers.wrap_commands_in_shell(
'windows', ['{}'.format(_PYTHON_INSTALL)]),
resource_files=[python_download],
wait_for_success=True,
user_identity=user_id)),
keep_alive=False,
pool_lifetime_option=batchmodels.PoolLifetimeOption.job))
sas_url = common.helpers.upload_blob_and_create_sas(
block_blob_client, _CONTAINER_NAME, _SIMPLE_TASK_NAME,
_SIMPLE_TASK_PATH,
datetime.datetime.utcnow() + datetime.timedelta(minutes=30))
job_spec = batchmodels.JobSpecification(
pool_info=pool_info,
# Terminate job once all tasks under it are complete to allow for a new
# job to be created under the schedule
on_all_tasks_complete=batchmodels.OnAllTasksComplete.terminate_job,
job_manager_task=batchmodels.JobManagerTask(
id="JobManagerTask",
command_line=common.helpers.wrap_commands_in_shell(
'windows', ['python {}'.format(_SIMPLE_TASK_NAME)]),
resource_files=[
batchmodels.ResourceFile(file_path=_SIMPLE_TASK_NAME,
http_url=sas_url)
]))
do_not_run_after = datetime.datetime.utcnow() \
+ datetime.timedelta(minutes=30)
schedule = batchmodels.Schedule(
do_not_run_after=do_not_run_after,
recurrence_interval=datetime.timedelta(minutes=10))
scheduled_job = batchmodels.JobScheduleAddParameter(
id=job_schedule_id, schedule=schedule, job_specification=job_spec)
batch_client.job_schedule.add(cloud_job_schedule=scheduled_job)
def create_job_schedule(batch_client, job_schedule_id, vm_size, vm_count):
"""Creates an Azure Batch pool and job schedule with the specified ids.
:param batch_client: The batch client to use.
:type batch_client: `batchserviceclient.BatchServiceClient`
:param str job_schedule_id: The id of the job schedule to create
:param str vm_size: vm size (sku)
:param int vm_count: number of vms to allocate
"""
pool_info = batchmodels.PoolInformation(
auto_pool_specification=batchmodels.AutoPoolSpecification(
auto_pool_id_prefix="JobScheduler",
pool=batchmodels.PoolSpecification(
vm_size=vm_size,
target_dedicated_nodes=vm_count,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=batchmodels.ImageReference(
publisher="Canonical",
offer="UbuntuServer",
sku="18.04-LTS",
version="latest"
),
node_agent_sku_id="batch.node.ubuntu 18.04"
),
start_task=batchmodels.StartTask(
command_line="/bin/bash -c "
"\"$AZ_BATCH_APP_PACKAGE_azure_batch_1/azure_batch/job_schedular_node_startup_tasks.sh\"",
wait_for_success=True,
user_identity=batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)
),
),
application_package_references=[batchmodels.ApplicationPackageReference(
application_id="azure_batch", version="1"
)],
),
keep_alive=False,
pool_lifetime_option=batchmodels.PoolLifetimeOption.job
)
)
job_spec = batchmodels.JobSpecification(
pool_info=pool_info,
# Terminate job once all tasks under it are complete to allow for a new
# job to be created under the schedule
on_all_tasks_complete=batchmodels.OnAllTasksComplete.terminate_job,
job_manager_task=batchmodels.JobManagerTask(
id="JobManagerTask",
#specify the command that needs to run recursively in job_schedular
command_line="/bin/bash -c \" python3 "
"$AZ_BATCH_APP_PACKAGE_azure_batch_1/azure_batch/azure_batch_main.py\""
))
#mention the interval of the job schedular
schedule = batchmodels.Schedule(
recurrence_interval=datetime.timedelta(days=15))
scheduled_job = batchmodels.JobScheduleAddParameter(
id=job_schedule_id,
schedule=schedule,
job_specification=job_spec)
batch_client.job_schedule.add(cloud_job_schedule=scheduled_job)
