def add_tasks(batch_service_client, job_id, task_id, number_to_test):
"""
Adds a task for each input file in the collection to the specified job.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str job_id: The ID of the job to which to add the tasks.
created for each input file.
:number_to_test: number you want to know if it's prime or not.
"""
print('Adding tasks to job [{}]...'.format(job_id))
# This is the user who run the command inside the container.
# An unprivileged one
user = batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.task,
elevation_level=batchmodels.ElevationLevel.non_admin)
# This is the docker image we want to run
task_container_settings = batchmodels.TaskContainerSettings(
image_name=config._DOCKER_IMAGE, container_run_options='--rm')
# The container needs this argument to be executed
task = batchmodels.TaskAddParameter(
id=task_id,
command_line='python /is_prime.py ' + str(number_to_test),
container_settings=task_container_settings,
user_identity=batchmodels.UserIdentity(auto_user=user))
batch_service_client.task.add(job_id, task)
def test_batch_applications(self, batch_job, **kwargs):
client = self.create_sharedkey_client(**kwargs)
# Test List Applications
apps = list(client.application.list())
self.assertEqual(len(apps), 1)
# Test Get Application
app = client.application.get('application_id')
self.assertIsInstance(app, models.ApplicationSummary)
self.assertEqual(app.id, 'application_id')
self.assertEqual(app.versions, ['v1.0'])
# Test Create Task with Application Package
task_id = 'python_task_with_app_package'
task = models.TaskAddParameter(
task_id,
'cmd /c "echo hello world"',
application_package_references=[models.ApplicationPackageReference('application_id', 'v1.0')]
)
response = client.task.add(batch_job.id, task)
self.assertIsNone(response)
# Test Get Task with Application Package
task = client.task.get(batch_job.id, task_id)
self.assertIsInstance(task, models.CloudTask)
self.assertEqual(task.application_package_references[0].application_id, 'application_id')
def configure_task(
self,
task_id: str,
command_line: str,
display_name: Optional[str] = None,
container_settings=None,
**kwargs,
) -> TaskAddParameter:
"""
Creates a task
:param task_id: A string that identifies the task to create
:param command_line: The command line of the Task.
:param display_name: A display name for the Task
:param container_settings: The settings for the container under which the Task runs.
If the Pool that will run this Task has containerConfiguration set,
this must be set as well. If the Pool that will run this Task doesn't have
containerConfiguration set, this must not be set.
"""
task = batch_models.TaskAddParameter(
id=task_id,
command_line=command_line,
display_name=display_name,
container_settings=container_settings,
**kwargs,
)
self.log.info("Task created: %s", task_id)
return task
def submit_job_and_add_task(batch_client, job_id, vm_size, vm_count):
"""Submits a job to the Azure Batch service
and adds a simple task with preparation task
:param batch_client: The batch client to use.
:type batch_client: `batchserviceclient.BatchServiceClient`
:param str job_id: The id of the job to create.
"""
pool_info = batchmodels.PoolInformation(
auto_pool_specification=batchmodels.AutoPoolSpecification(
auto_pool_id_prefix="Helloworld_jobprep",
pool=batchmodels.PoolSpecification(
vm_size=vm_size,
target_dedicated_nodes=vm_count,
cloud_service_configuration={'os_family': "4"}),
keep_alive=False,
pool_lifetime_option=batchmodels.PoolLifetimeOption.job))
job = batchmodels.JobAddParameter(
id=job_id,
pool_info=pool_info,
job_preparation_task=batch.models.JobPreparationTask(
command_line=prep_task_command, wait_for_success=True))
batch_client.job.add(job)
task = batchmodels.TaskAddParameter(
id="HelloWorld_Task",
command_line=common.helpers.wrap_commands_in_shell(
'windows',
['echo Hello world from the Batch Hello world sample!']))
batch_client.task.add(job_id=job.id, task=task)
def add_tasks(batch_service_client, job_id, task_id):
"""
Adds a task for each input file in the collection to the specified job.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str job_id: The ID of the job to which to add the tasks.
created for each input file.
:number_to_test: number you want to know if it's prime or not.
"""
print('Adding tasks to job [{}]...'.format(job_id))
# This is the user who run the command inside the container.
# An unprivileged one
user = batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.task,
elevation_level=batchmodels.ElevationLevel.non_admin)
# This is the docker image we want to run
task_container_settings = batchmodels.TaskContainerSettings(
image_name=config._DOCKER_IMAGE,
container_run_options='--rm -e PYTHONUNBUFFERED=1')
# The container needs this argument to be executed
# remember we run the container like: docker ... imagename python /is_prime.py number
task = batchmodels.TaskAddParameter(
id=task_id,
command_line=
"python main.py quantity --job-id=2 --keyword='covid19' --question='Is covid19 man made' --num-papers=10",
container_settings=task_container_settings,
user_identity=batchmodels.UserIdentity(auto_user=user))
batch_service_client.task.add(job_id, task)
def add_docker_batch_task(batch_client, block_blob_client, job_id, pool_id):
"""Submits a docker task via Batch scheduler
:param batch_client: The batch client to use.
:type batch_client: `batchserviceclient.BatchServiceClient`
:param block_blob_client: The storage block blob client to use.
:type block_blob_client: `azure.storage.blob.BlockBlobService`
:param str job_id: The id of the job to use.
:param str pool_id: The id of the pool to use.
:rtype: str
:return: task id of added task
"""
task_commands = [
'docker run hello-world',
'docker ps -a',
]
task = batchmodels.TaskAddParameter(
id=_TASK_ID,
command_line=common.helpers.wrap_commands_in_shell(
'linux', task_commands),
run_elevated=True,
)
batch_client.task.add(job_id=job_id, task=task)
return task.id
def add_task_to_start_server(self):
constraint = batchmodels.TaskConstraints(
retention_time=datetime.timedelta(hours=24), )
user_identity = batch.models.UserIdentity(
# user_name='azureuser',
auto_user=batch.models.AutoUserSpecification(
scope=batch.models.AutoUserScope.pool,
elevation_level=batch.models.ElevationLevel.admin))
task = batchmodels.TaskAddParameter(
id=helpers.generate_unique_resource_name(
f"Server-{str(self.globalTaskCounter)}"),
# id=f"Server-{str(self.globalTaskCounter)}",
command_line=helpers.wrap_commands_in_shell(
'linux', self.get_start_task_commands()),
constraints=constraint,
user_identity=user_identity)
try:
self.globalTaskCounter += 1
self.client.task.add(job_id=self.job_id, task=task)
except BatchErrorException as e:
self.globalTaskCounter += 1
self.start_mc_server_job_pool(1)
self.client.task.add(job_id=self.job_id, task=task)
def do_dockerstuff(batch_client, block_blob_client, job_id, pool_id,
docker_user, docker_password, docker_image):
# upload scripts
config_url = common.helpers.upload_blob_and_create_sas(
block_blob_client, _CONTAINER_NAME, 'config.txt',
os.path.join('resources', 'config.txt'),
datetime.datetime.utcnow() + datetime.timedelta(hours=1))
task_commands = [
'mkdir -p /opt/scripts/config',
'cd /opt/scripts/config',
'wget ' + common.helpers._read_stream_as_string(
'\"' + config_url + '\" -O /opt/scripts/config/config.txt',
'utf-8'),
'docker login chris-microsoft.azurecr.io -u ' + docker_user + ' -p ' +
docker_password,
'docker run -i -v `pwd`:/opt/scripts/config ' + docker_image +
' /opt/scripts/getdata.sh',
]
task_name = common.helpers.generate_unique_resource_name(_TASK_NAME)
print('adding task: ' + task_name)
task = batchmodels.TaskAddParameter(
id=task_name,
command_line=common.helpers.wrap_commands_in_shell(
'linux', task_commands),
run_elevated=True,
)
batch_client.task.add(job_id=job_id, task=task)
time.sleep(5)
return task.id
def run_commands(batch_client, block_blob_client, job_id, pool_id):
"""Run the start commands listed in the file "start_commands" on
all the nodes of the Azure Batch service.
:param batch_client: The batch client to use.
:type batch_client: `batchserviceclient.BatchServiceClient`
:param block_blob_client: The storage block blob client to use.
:type block_blob_client: `azure.storage.blob.BlockBlobService`
:param str job_id: The id of the job to create.
:param str pool_id: The id of the pool to use.
"""
task_commands = get_list_from_file('configs/start_commands')
logging.info(task_commands)
user = batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)
start = time.time()
job = batchmodels.JobAddParameter(
id=job_id, pool_info=batchmodels.PoolInformation(pool_id=pool_id))
batch_client.job.add(job)
logging.info('job created in seconds {}'.format(time.time() - start))
start = time.time()
nodes = list(batch_client.compute_node.list(pool_id))
tasks = [batchmodels.TaskAddParameter(
id="EBOTask-{}".format(i),
command_line=common.helpers.wrap_commands_in_shell('linux', task_commands),
user_identity=batchmodels.UserIdentity(auto_user=user)) \
for i in xrange(len(nodes))]
batch_client.task.add_collection(job.id, tasks)
logging.info('task created in seconds {}'.format(time.time() - start))
def submit_job_and_add_task(batch_client, block_blob_client, job_id, pool_id):
"""Submits a job to the Azure Batch service and adds
a task that runs a python script.
:param batch_client: The batch client to use.
:type batch_client: `batchserviceclient.BatchServiceClient`
:param block_blob_client: The storage block blob client to use.
:type block_blob_client: `azure.storage.blob.BlockBlobService`
:param str job_id: The id of the job to create.
:param str pool_id: The id of the pool to use.
"""
job = batchmodels.JobAddParameter(
id=job_id, pool_info=batchmodels.PoolInformation(pool_id=pool_id))
batch_client.job.add(job)
block_blob_client.create_container(_CONTAINER_NAME, fail_on_exist=False)
sas_url = common.helpers.upload_blob_and_create_sas(
block_blob_client, _CONTAINER_NAME, _SIMPLE_TASK_NAME,
_SIMPLE_TASK_PATH,
datetime.datetime.utcnow() + datetime.timedelta(hours=1))
task = batchmodels.TaskAddParameter(
id="MyPythonTask",
command_line="python " + _SIMPLE_TASK_NAME,
resource_files=[
batchmodels.ResourceFile(file_path=_SIMPLE_TASK_NAME,
http_url=sas_url)
])
batch_client.task.add(job_id=job.id, task=task)
def add_tasks(self, part_file, JOB_ID):
"""
Adds a task for each input file in the collection to the specified job.
"""
# print("Adding {} tasks to job [{}]...".format(count, job_id))
tasks = list()
for i in range(self.K):
output_file = self.build_output_file(i)
command_line = "/bin/bash -c 'echo $AZ_BATCH_TASK_WORKING_DIR && daemon status && scgrad {} {} {} {}'".format(
_GRAD_COMMON_FILE, _GRAD_PART_FILE, _CONTAINER_OUTPUT_FILE, i)
if self.config.REGISTRY_USERNAME:
registry = models.ContainerRegistry(
user_name=self.config.REGISTRY_USERNAME,
password=self.config.REGISTRY_PASSWORD,
registry_server=self.config.REGISTRY_SERVER,
)
task_container_settings = models.TaskContainerSettings(
image_name=self.config.DOCKER_IMAGE, registry=registry)
else:
task_container_settings = models.TaskContainerSettings(
image_name=self.config.DOCKER_IMAGE)
tasks.append(
models.TaskAddParameter(
id="grad_part_{}".format(i),
command_line=command_line,
resource_files=[self.common_file, part_file],
output_files=[output_file],
container_settings=task_container_settings,
))
self.batch_client.task.add_collection(JOB_ID, [tasks[0]])
def add_task(
self,
resource_files: List[models.ResourceFile],
output_files: List[models.OutputFile],
command_line=None,
):
"""
Adds a task for each input file in the collection to the specified job.
:param list resource_files: A list of ResouceFile descriptions for the task
:param list output_files: A list of OutputFile descriptions for the task
:param str command_line: The command used to for the task. Optional;
if missing, defaults to the command_line parameter provided when
instantiating this object
"""
self.tasks.append(
models.TaskAddParameter(
id="Task_{}".format(len(self.tasks)),
command_line=self.config.COMMAND_LINE
if command_line is None else command_line,
resource_files=resource_files,
output_files=output_files,
container_settings=models.TaskContainerSettings(
image_name=self.config.DOCKER_IMAGE),
))
def add_task(batch_service_client, job_id, task_id, application_cmdline,
input_files, run_elevated, num_instances, coordination_cmdline,
common_files):
"""
Adds a task for each input file in the collection to the specified job.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str job_id: The ID of the job to which to add the task.
:param str task_id: The ID of the task to be added.
:param str application_cmdline: The application commandline for the task.
:param list input_files: A collection of input files.
:param bool run_elevated: flag determining if task should run as elevated
:param int num_instances: Number of instances for the task
:param str coordination_cmdline: The application commandline for the task.
:param list common_files: A collection of common input files.
"""
print('Adding {} task to job [{}]...'.format(task_id, job_id))
multi_instance_settings = None
if coordination_cmdline or (num_instances and num_instances > 1):
multi_instance_settings = batchmodels.MultiInstanceSettings(
number_of_instances=num_instances,
coordination_command_line=coordination_cmdline,
common_resource_files=common_files)
task = batchmodels.TaskAddParameter(
id=task_id,
command_line=application_cmdline,
run_elevated=run_elevated,
resource_files=input_files,
multi_instance_settings=multi_instance_settings)
batch_service_client.task.add(job_id, task)
def create_task(batch_client, name_job, cmd, name_task, param_multi_inst=None):
current_date = localtime()[0:5]
current_date = "{0}{1}{2}{3}{4}".format(current_date[0], current_date[1],
current_date[2], current_date[3],
current_date[4])
dest_files_in_container = models.OutputFileBlobContainerDestination(
container_url=
f"{config_resources.blob_container['url']}{config_resources.blob_container['sas_token']}",
path=f"{name_job}/{name_task}/{current_date}")
dest_files = models.OutputFileDestination(
container=dest_files_in_container)
trigger_upload = models.OutputFileUploadOptions(
upload_condition='taskCompletion')
upload_files = models.OutputFile(file_pattern='$AZ_BATCH_TASK_DIR/*',
destination=dest_files,
upload_options=trigger_upload)
outputs = []
outputs.append(upload_files)
tache = models.TaskAddParameter(id=name_task,
command_line=cmd,
multi_instance_settings=param_multi_inst,
resource_files=None,
environment_settings=None,
output_files=outputs)
batch_client.task.add(name_job, tache)
def submit_job_and_add_task(batch_client, block_blob_client,
storage_account_name, storage_account_key,
container, resourcefile, job_id, pool_id,
sha1_cert_tp):
"""Submits a job to the Azure Batch service and adds
a task that decrypts the file stored in Azure Storage.
:param batch_client: The batch client to use.
:type batch_client: `batchserviceclient.BatchServiceClient`
:param block_blob_client: The storage block blob client to use.
:type block_blob_client: `azure.storage.blob.BlockBlobService`
:param str storage_account_name: storage account name
:param str storage_account_key: storage account key
:param str container: blob storage container
:param str resourcefile: resource file to add to task
:param str job_id: The id of the job to create.
:param str pool_id: The id of the pool to use.
:param str sha1_cert_tp: sha1 cert thumbprint for cert ref
"""
job = batchmodels.JobAddParameter(
id=job_id, pool_info=batchmodels.PoolInformation(pool_id=pool_id))
batch_client.job.add(job)
# generate short-lived sas key for blobxfer
sastoken = common.helpers.create_sas_token(block_blob_client, container,
_RESOURCE_NAME,
azureblob.BlobPermissions.READ)
# issue the following commands for the task:
# 1. convert pfx installed by the Azure Batch Service to pem
# 2. transfer the encrypted blob from Azure Storage to local disk and
# decrypt contents using the private key
# 3. output decrypted secret.txt file
# Note: certs on Linux Batch Compute Nodes are placed in:
# $AZ_BATCH_CERTIFICATES_DIR where the cert itself has a naming convention
# of <thumbprint algorithm>-<lowercase thumbprint>.<certificate format>
task_commands = [
('openssl pkcs12 -in $AZ_BATCH_CERTIFICATES_DIR/sha1-{tp}.pfx -out '
'$AZ_BATCH_CERTIFICATES_DIR/privatekey.pem -nodes -password '
'file:$AZ_BATCH_CERTIFICATES_DIR/sha1-{tp}.pfx.pw').format(
tp=sha1_cert_tp),
('blobxfer {sa} {cont} . --saskey "{sas}" --download '
'--remoteresource {rf} --rsaprivatekey '
'$AZ_BATCH_CERTIFICATES_DIR/privatekey.pem').format(
sa=storage_account_name,
cont=container,
sas=sastoken,
rf=resourcefile), 'echo secret.txt contents:',
'cat {}'.format(resourcefile)
]
task = batchmodels.TaskAddParameter(
id="MyEncryptedResourceTask",
command_line=common.helpers.wrap_commands_in_shell(
'linux', task_commands),
)
batch_client.task.add(job_id=job.id, task=task)
def designate_master_docker_swarm_node(batch_client, pool_id, nodes, job_id):
"""Designate a master docker swarm node by selecting a node in the
pool to be the swarm manager. This is accomplished via IP selection in
the pool of nodes and running the swarm init command via an
affinitized task. This is for Docker 1.12+.
:param batch_client: The batch client to use.
:type batch_client: `batchserviceclient.BatchServiceClient`
:param str pool_id: The id of the pool.
:param list nodes: list of `batchserviceclient.models.ComputeNode`
:param str job_id: The id of the job to create.
:rtype: tuple
:return: ((master ipaddress, master node id), swarm token)
"""
# designate the lowest ip address node as the master
nodes = sorted(nodes, key=lambda node: node.ip_address)
master_node_ip_address = nodes[0].ip_address
master_node_id = nodes[0].id
master_node_affinity_id = nodes[0].affinity_id
master_node = (master_node_ip_address, master_node_id)
print('master node is: {}'.format(master_node))
# create job
job = batchmodels.JobAddParameter(
id=job_id, pool_info=batchmodels.PoolInformation(pool_id=pool_id))
batch_client.job.add(job)
# add docker swarm manage as an affinitized task to run on the master node
# NOTE: task affinity is weak. if the node has no available scheduling
# slots, the task may be executed on a different node. for this example,
# it is not an issue since this node should be available for scheduling.
task_commands = [
'docker swarm init --advertise-addr {}'.format(master_node_ip_address),
'docker swarm join-token -q worker',
]
print('initializing docker swarm cluster via Azure Batch task...')
task = batchmodels.TaskAddParameter(
id='swarm-manager',
affinity_info=batchmodels.AffinityInformation(
affinity_id=master_node_affinity_id),
command_line=common.helpers.wrap_commands_in_shell(
'linux', task_commands),
run_elevated=True,
)
batch_client.task.add(job_id=job.id, task=task)
# wait for task to complete
common.helpers.wait_for_tasks_to_complete(batch_client, job_id,
datetime.timedelta(minutes=5))
# retrieve the swarm token
stdout = common.helpers.read_task_file_as_string(
batch_client, job.id, task.id, common.helpers._STANDARD_OUT_FILE_NAME)
token = stdout.splitlines()[-1].strip()
print('swarm token: {}'.format(token))
return master_node, token
def add_docker_batch_task(batch_client, block_blob_client, job_id, pool_id):
"""Submits a docker task via Batch scheduler
:param batch_client: The batch client to use.
:type batch_client: `batchserviceclient.BatchServiceClient`
:param block_blob_client: The storage block blob client to use.
:type block_blob_client: `azure.storage.blob.BlockBlobService`
:param str job_id: The id of the job to use.
:param str pool_id: The id of the pool to use.
:rtype: list
:return: a list of task_id of the task added.
"""
task_resource_sas_url = common.helpers.upload_blob_and_create_sas(
block_blob_client, _CONTAINER_NAME, _TASK_RESOURCE_FILE,
_TASK_RESOURCE_FILE_PATH,
datetime.datetime.utcnow() + datetime.timedelta(hours=1))
output_container_sas_key = common.helpers.create_container_and_create_sas(
block_blob_client=block_blob_client,
container_name=_OUTPUT_CONTAINER_NAME,
permission=azureblob.ContainerPermissions.WRITE
| azureblob.ContainerPermissions.LIST,
expiry=datetime.datetime.utcnow() + datetime.timedelta(hours=1))
# The start task pulls docker image yidingz/ffmpeg:v3
job = batchmodels.JobAddParameter(
id=job_id,
pool_info=batchmodels.PoolInformation(pool_id=pool_id),
job_preparation_task=batchmodels.JobPreparationTask(
command_line=_JOB_STARTTASK_CLI, run_elevated=True))
batch_client.job.add(job)
task_id_list = []
index = 0
for url in _INPUT_FILE_URLS:
filename = urllib.parse.urlsplit(url).path.split('/')[-1]
parameters = "'{0}' '{1}' '{2}' '{3}'".format(
url, filename, output_container_sas_key,
block_blob_client.account_name)
# Each task will download a video from chanel9,
# transcode, and upload to specified output container
task = batchmodels.TaskAddParameter(
id=str(index).zfill(4) + '_' + filename.split('.')[0],
command_line=_TASK_CLI.format(_TASK_RESOURCE_FILE, _FFMPEG_IMAGE,
parameters),
run_elevated=True,
resource_files=[
batchmodels.ResourceFile(file_path=_TASK_RESOURCE_FILE,
blob_source=task_resource_sas_url)
])
task_id_list.append(task.id)
batch_client.task.add(job_id=job_id, task=task)
index += 1
return task_id_list
def designate_master_docker_swarm_node(batch_client, pool_id, nodes, job_id):
"""Designate a master docker swarm node by selecting a node in the
pool to be the swarm manager. This is accomplished via IP selection in
the pool of nodes and running the swarm manage command via an
affinitized task.
:param batch_client: The batch client to use.
:type batch_client: `batchserviceclient.BatchServiceClient`
:param str pool_id: The id of the pool to create.
:param list nodes: list of `batchserviceclient.models.ComputeNode`
:param str job_id: The id of the job to create.
:rtype: tuple
:return: (ipaddress, master node id)
"""
# designate the lowest ip address node as the master
nodes = sorted(nodes, key=lambda node: node.ip_address)
master_node_ip_address = nodes[0].ip_address
master_node_id = nodes[0].id
master_node = (master_node_ip_address, master_node_id)
# create a node list based on the number of nodes and master
lastoctet = int(master_node_ip_address.split('.')[-1])
nodelist = '10.0.0.[{}:{}]'.format(lastoctet, lastoctet + len(nodes) - 1)
print('master node is: {} nodelist is: {}'.format(master_node, nodelist))
# create job
job = batchmodels.JobAddParameter(
id=job_id,
pool_info=batchmodels.PoolInformation(pool_id=pool_id))
batch_client.job.add(job)
# add docker swarm manage as an affinitized task to run on the master node
task_commands = [
('docker run -d -p 3375:3375 -t swarm manage -H tcp://0.0.0.0:3375 '
'"nodes://{}:2375"').format(nodelist)
]
print('creating docker swarm cluster via Azure Batch task...')
task = batchmodels.TaskAddParameter(
id="swarm-master",
affinity_info=batchmodels.AffinityInformation(
affinity_id=master_node_id),
command_line=common.helpers.wrap_commands_in_shell(
'linux', task_commands),
run_elevated=True,
)
batch_client.task.add(job_id=job.id, task=task)
# wait for task to complete
common.helpers.wait_for_tasks_to_complete(
batch_client,
job_id,
datetime.timedelta(minutes=5))
print('docker swarm cluster created.')
return master_node
def add_task(self, job_id, task_id, input_file, job_shell_files,
job_script_file, output_blob_prefix):
"""
Adds a task for each input file in the collection to the specified job.
:param job_id: The ID of the job to which to add the tasks.
:type job_id: str
:param task_id: The ID of the task.
:type task_id: str
:param input_file: delegator input file
:type input_file: str
:param job_script_file: job.sh file
:type job_script_file: str
:param output_blob_prefix: prefix where output files will be uploaded to
:type output_blob_prefix: str
"""
commands = ['/bin/bash job.sh']
resource_files = [input_file, job_script_file] + job_shell_files
# Obtain a shared access signature that provides write access to the output
# container to which the tasks will upload their output.
output_sas_url = self.blob_client.generate_container_url(
container_name=self.container_name,
add_sas=True,
sas_permissions='create|write')
def create_output_file(filename, blobname):
container = batchmodels.OutputFileBlobContainerDestination(
container_url=output_sas_url, path=blobname)
destination = batchmodels.OutputFileDestination(
container=container)
upload_options = batchmodels.OutputFileUploadOptions(
upload_condition='taskCompletion')
return batchmodels.OutputFile(file_pattern=filename,
destination=destination,
upload_options=upload_options)
task = batchmodels.TaskAddParameter(
id=task_id,
command_line=self.__wrap_commands_in_shell('linux', commands),
resource_files=resource_files,
output_files=[
create_output_file(
'../stdout.txt',
os.path.join(output_blob_prefix, 'stdout.txt')),
create_output_file(
'../stderr.txt',
os.path.join(output_blob_prefix, 'stderr.txt')),
create_output_file(
'job_result.pickle',
os.path.join(output_blob_prefix, 'job_result.pickle')),
])
self.batch_client.task.add(job_id, task)
def create_post_processing_task(container_settings, job_id, tasks):
post_processing_task = batchmodels.TaskAddParameter(
id="postprocessing",
command_line=f'/bin/sh -c "cat {job_id}/output/*.csv"',
# oddly, using storage_container_url doesn't work here
# but storage container name does
resource_files=[batchmodels.ResourceFile(auto_storage_container_name=_CONTAINER_NAME,
blob_prefix=f"{job_id}/output/",
file_path="")],
container_settings=container_settings,
depends_on=batchmodels.TaskDependencies(task_ids=[task.id for task in tasks])
)
return post_processing_task
def submit_job_and_add_tasks(batch_client, block_blob_client, job_id, pool_id,
in_files, out_container_name, app_files,
storage_account_name, out_sas_token):
"""Submits jobs to the Azure Batch service and adds
tasks that runs a python script.
:param batch_client: The batch client to use.
:type batch_client: `batchserviceclient.BatchServiceClient`
:param block_blob_client: The storage block blob client to use.
:type block_blob_client: `azure.storage.blob.BlockBlobService`
:param str job_id: The id of the job to create.
:param str pool_id: The id of the pool to use.
:param list in_files: The list of the file paths of the inputs.
:param str out_container_name: The name of the output container.
:param list app_files: The list of all the other scripts to upload.
:param str storage_account_name: The name of the storage account.
:param str out_sas_token: A SAS token granting the specified
permissions to the output container.
"""
start = time.time()
job = batchmodels.JobAddParameter(
id=job_id, pool_info=batchmodels.PoolInformation(pool_id=pool_id))
batch_client.job.add(job)
logging.info('job created in seconds {}'.format(time.time() - start))
start = time.time()
tasks = [batchmodels.TaskAddParameter(
id="EBOTask-{}".format(i),
command_line='python {} --filepath {} --storageaccount {} --storagecontainer {} --sastoken "{}"'.format(
_TASK_FILE,
in_file.file_path,
storage_account_name,
out_container_name,
out_sas_token),
resource_files=[in_file] + app_files) \
for i, in_file in enumerate(in_files)]
cnt = 0
tot_tasks = len(tasks)
while cnt < tot_tasks:
try:
batch_client.task.add_collection(job.id, tasks[cnt:cnt + 100])
cnt += 100
except Exception as e:
print("Adding task failed... Going to try again in 5 seconds")
logging.error(e)
time.sleep(5)
logging.info('task created in seconds {}'.format(time.time() - start))
def create_processing_tasks(block_blob_client, job_id, container_settings):
tasks = []
resource_files = []
container_url = azure_helpers.create_container_sas(
block_blob_client,
_CONTAINER_NAME,
datetime.datetime.utcnow() + datetime.timedelta(hours=24)
)
for task_script in ["petal_width.py", "sepal_width.py"]:
task_name = task_script.replace("_", "")
task_path = Path("resources", task_script)
script_stem = task_script.rstrip(".py")
# Shared access signature, allows access to a resource using a token for a specified amount of time
sas_url = azure_helpers.upload_blob_and_create_sas(
block_blob_client,
_CONTAINER_NAME,
task_name,
task_path,
datetime.datetime.utcnow() + datetime.timedelta(hours=1))
resource_file = batchmodels.ResourceFile(
file_path=task_name,
http_url=sas_url)
resource_files.append(resource_file)
# output data to upload to storage container
output_file = batchmodels.OutputFile(
file_pattern=f'output/{script_stem}.csv',
destination=batchmodels.OutputFileDestination(
container=batchmodels.OutputFileBlobContainerDestination(
container_url=container_url,
path=f'{job_id}/output/{script_stem}.csv'
)
),
upload_options=batchmodels.OutputFileUploadOptions(
upload_condition=batchmodels.OutputFileUploadCondition.task_completion
)
)
# add tasks with resources and outputs
tasks.append(batchmodels.TaskAddParameter(
id=script_stem,
command_line=f'python3 {task_name}',
resource_files=[resource_file],
container_settings=container_settings,
output_files=[output_file]
))
return resource_files, tasks
def add_task(
batch_service_client, job_id, task_id, num_instances,
application_cmdline, input_files, elevation_level,
output_file_names, output_container_sas,
coordination_cmdline, common_files):
"""
Adds a task for each input file in the collection to the specified job.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str job_id: The ID of the job to which to add the task.
:param str task_id: The ID of the task to be added.
:param str application_cmdline: The application commandline for the task.
:param list input_files: A collection of input files.
:param elevation_level: Elevation level used to run the task; either
'admin' or 'nonadmin'.
:type elevation_level: `azure.batch.models.ElevationLevel`
:param int num_instances: Number of instances for the task
:param str coordination_cmdline: The application commandline for the task.
:param list common_files: A collection of common input files.
"""
print('Adding {} task to job [{}]...'.format(task_id, job_id))
multi_instance_settings = None
if coordination_cmdline or (num_instances and num_instances > 1):
multi_instance_settings = batchmodels.MultiInstanceSettings(
number_of_instances=num_instances,
coordination_command_line=coordination_cmdline,
common_resource_files=common_files)
user = batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=elevation_level)
output_file = batchmodels.OutputFile(
file_pattern=output_file_names,
destination=batchmodels.OutputFileDestination(
container=batchmodels.OutputFileBlobContainerDestination(
container_url=output_container_sas)),
upload_options=batchmodels.OutputFileUploadOptions(
upload_condition=batchmodels.
OutputFileUploadCondition.task_completion))
task = batchmodels.TaskAddParameter(
id=task_id,
command_line=application_cmdline,
user_identity=batchmodels.UserIdentity(auto_user=user),
resource_files=input_files,
multi_instance_settings=multi_instance_settings,
output_files=[output_file])
batch_service_client.task.add(job_id, task)
def create_job(self, job_id, pool_id, total_nodes, is_linux_pool):
client = self._get_batch_client()
try:
pool_info = batchmodels.PoolInformation(pool_id=pool_id)
job = batchmodels.JobAddParameter(id=job_id, pool_info=pool_info)
try:
client.job.add(job)
except batchmodels.BatchErrorException as be:
if be.error and be.error.code == 'JobExists':
pass
else:
print('Error creating job, code={}, message={}'.format(
be.error.code, be.error.message))
raise
if is_linux_pool:
cmd_line = '/bin/bash -c azure-batch-ses.sh'
script = 'azure-batch-ses.sh'
script_url = 'https://raw.githubusercontent.com/Azure/azure-deadline/master/CloudProviderPlugin/Scripts/azure-batch-ses.sh'
else:
cmd_line = 'powershell.exe -file azure-batch-ses.ps1'
script = 'azure-batch-ses.ps1'
script_url = 'https://raw.githubusercontent.com/Azure/azure-deadline/master/CloudProviderPlugin/Scripts/azure-batch-ses.ps1'
task = batchmodels.TaskAddParameter(
id='',
command_line=cmd_line,
resource_files=[batchmodels.ResourceFile(script_url, script)],
constraints=batchmodels.TaskConstraints(
max_task_retry_count=3),
user_identity=batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)))
for i in range(total_nodes):
task.id = str(uuid.uuid4())
client.task.add(job_id=job.id, task=task)
except batchmodels.BatchErrorException as be:
if be.error:
print('Error creating job, code={}, message={}'.format(
be.error.code, be.error.message))
if be.error.values:
for e in be.error.values:
print('Key={}, Value={}'.format(e.key, e.value))
raise
def add_tasks(batch_service_client, job_id, input_files,
output_container_sas_url):
"""Adds a task for each pair with input file and command line in the collection to the specified job.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str job_id: The ID of the job to which to add the tasks.
:param list input_files: A collection of input files.
:param output_container_sas_url: A SAS token granting write access to
the specified Azure Blob storage container."""
print('Adding [{}] tasks to job [{}]'.format(_MAX_TASKS, job_id))
tasks = list()
mcr = "".join(input_files[-4].file_path)
program = "".join(input_files[-3].file_path)
data = "".join(input_files[-2].file_path)
design = "".join(input_files[-1].file_path)
output_file_pattern = '*.mat'
cogns = ['cogn_ps', 'cogn_po']
for idx in range(1, _MAX_TASKS + 1):
command = "/bin/bash -c 'sudo apt-get update -qq && sudo apt-get install libxt6 default-jre -y;tar xf {};" \
"mcr/install -destinationFolder /tmp/mcr18b -mode silent -agreeToLicense yes;rm -rf mcr {} /tmp/ma*" \
";./{} /tmp/mcr18b/v95 {} {} {} {} {} 15'".format(
mcr, mcr, program, data, design, cogns[1], 'batch' + str(idx) + '.mat', idx)
tasks.append(
batchmodels.TaskAddParameter(
id='Task{}'.format(idx),
command_line=command,
resource_files=input_files,
output_files=[
batchmodels.OutputFile(
file_pattern=output_file_pattern,
destination=batchmodels.OutputFileDestination(
container=batchmodels.
OutputFileBlobContainerDestination(
container_url=output_container_sas_url)),
upload_options=batchmodels.OutputFileUploadOptions(
upload_condition='taskCompletion'))
],
user_identity=batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin))))
batch_service_client.task.add_collection(job_id, tasks)
def add_tasks(
config,
_blob_client,
batch_service_client,
container_sas_url,
job_id,
_input_file,
count,
):
"""
Adds a task for each input file in the collection to the specified job.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str job_id: The ID of the job to which to add the tasks.
:param list input_files: The input files
:param output_container_sas_token: A SAS token granting write access to
the specified Azure Blob storage container.
"""
print("Adding {} tasks to job [{}]...".format(count, job_id))
tasks = list()
for fold_number in range(count):
output_file = build_output_file(container_sas_url, fold_number)
# command_line = '/bin/bash -c \'echo "Hello World" && echo "hello: world" > output.yaml\''
command_line = "/bin/bash -c 'stt {} {} {}'".format(
_CONTAINER_INPUT_FILE, _CONTAINER_OUTPUT_FILE, fold_number)
task_container_settings = models.TaskContainerSettings(
image_name=config.DOCKER_CONTAINER)
tasks.append(
models.TaskAddParameter(
id="Task_{}".format(fold_number),
command_line=command_line,
resource_files=[_input_file],
output_files=[output_file],
container_settings=task_container_settings,
))
batch_service_client.task.add_collection(job_id, tasks)
def add_tasks(batch_service_client, job_id, input_files,
output_container_sas_url):
"""Adds a task for each pair with input file and command line in the collection to the specified job.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str job_id: The ID of the job to which to add the tasks.
:param list input_files: A collection of input files. One task will be
created for each input file
:param output_container_sas_url: A SAS token granting write access to
the specified Azure Blob storage container."""
print('Adding [{}] tasks to job [{}]'.format(len(input_files) - 2, job_id))
tasks = list()
output_file_pattern = '*-[01w]*'
template = "".join(input_files[-1].file_path.split('.')[:-2])
script = "".join(input_files[-2].file_path)
for idx, input_file in enumerate(input_files[:-2]):
input_file_path = input_file.file_path
subjid = "".join(input_file_path.split('.')[:-2])
command = "/bin/bash -c 'source /usr/local/software/addpaths;./{} {} {}'".format(
script, template, subjid)
tasks.append(
batchmodels.TaskAddParameter(
id='Task{}'.format(idx + 1),
command_line=command,
resource_files=[input_file, input_files[-2], input_files[-1]],
output_files=[
batchmodels.OutputFile(
file_pattern=output_file_pattern,
destination=batchmodels.OutputFileDestination(
container=batchmodels.
OutputFileBlobContainerDestination(
container_url=output_container_sas_url)),
upload_options=batchmodels.OutputFileUploadOptions(
upload_condition='taskCompletion'))
],
user_identity=batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin))))
batch_service_client.task.add_collection(job_id, tasks)
def add_tasks(batch_service_client, job_id, input_files,
output_container_sas_url, num_tasks):
"""Adds a task for each pair with input file and command line in the collection to the specified job.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str job_id: The ID of the job to which to add the tasks.
:param list input_files: A collection of input files.
:param output_container_sas_url: A SAS token granting write access to
the specified Azure Blob storage container.
:param num_tasks: Total number of tasks."""
print('Adding [{}] tasks to job [{}]'.format(num_tasks, job_id))
tasks = list()
output_file_pattern = '[ts]*'
for idx in range(num_tasks):
# input_file_path = input_file.file_path
# subjid = "".join(input_file_path.split('.')[:-2])
command = "/bin/bash -c 'source /usr/local/software/addpaths;" \
"randomise -i all_FA_skeletonised -m mean_FA_skeleton_mask -d design.mat -t design.con " \
"-D --T2 --uncorrp --seed={} -n {} -o {}_SEED{};cp -p ../stdout.txt stdout{}.txt'" \
"".format(idx+1, int(_RANDOMISE_ITER/_NUM_TASKS), _OUTPUT_PREFIX, idx+1, idx+1)
tasks.append(
batchmodels.TaskAddParameter(
id='Task{}'.format(idx + 1),
command_line=command,
resource_files=input_files,
output_files=[
batchmodels.OutputFile(
file_pattern=output_file_pattern,
destination=batchmodels.OutputFileDestination(
container=batchmodels.
OutputFileBlobContainerDestination(
container_url=output_container_sas_url)),
upload_options=batchmodels.OutputFileUploadOptions(
upload_condition='taskCompletion'))
],
user_identity=batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin))))
batch_service_client.task.add_collection(job_id, tasks)
def schedule_with_target(
core_cluster_operations,
spark_cluster_operations,
cluster_id,
scheduling_target,
task,
wait,
internal,
):
# upload "real" task definition to storage
serialized_task_resource_file = upload_serialized_task_to_storage(
core_cluster_operations.blob_client, cluster_id, task)
# # schedule "ghost" task
ghost_task = batch_models.TaskAddParameter(
id=task.id,
command_line="/bin/bash",
)
# tell the node to run the task
core_cluster_operations.batch_client.task.add(cluster_id, task=ghost_task)
task_working_dir = "/mnt/aztk/startup/tasks/workitems/{}".format(task.id)
task_cmd = (
r"source ~/.bashrc; "
r"mkdir -p {0};"
r"export PYTHONPATH=$PYTHONPATH:$AZTK_WORKING_DIR; "
r"export AZ_BATCH_TASK_WORKING_DIR={0};"
r"export STORAGE_LOGS_CONTAINER={1};"
r"cd $AZ_BATCH_TASK_WORKING_DIR; "
r'$AZTK_WORKING_DIR/.aztk-env/.venv/bin/python $AZTK_WORKING_DIR/aztk/node_scripts/scheduling/submit.py "{2}" >> {3} 2>&1'
.format(task_working_dir, cluster_id,
serialized_task_resource_file.blob_source,
constants.SPARK_SUBMIT_LOGS_FILE))
node_id = select_scheduling_target_node(spark_cluster_operations,
cluster_id, scheduling_target)
node_run_output = spark_cluster_operations.node_run(cluster_id,
node_id,
task_cmd,
timeout=120,
block=wait,
internal=internal)
def submit_job_and_add_task(batch_client, block_blob_client, job_id, pool_id):
job = batchmodels.JobAddParameter(
id=job_id, pool_info=batchmodels.PoolInformation(pool_id=pool_id))
batch_client.job.add(job)
block_blob_client.create_container(CONTAINER_NAME, fail_on_exist=False)
sas_url = upload_blob_and_create_sas(
block_blob_client, CONTAINER_NAME, TASK_NAME, TASK_PATH,
datetime.datetime.utcnow() + datetime.timedelta(hours=1))
task = batchmodels.TaskAddParameter(id="SliceTask",
command_line="python3 " + TASK_NAME,
resource_files=[
batchmodels.ResourceFile(
file_path=TASK_NAME,
blob_source=sas_url)
])
batch_client.task.add(job_id=job.id, task=task)