def create_task(batch_client, name_job, cmd, name_task, param_multi_inst=None):
current_date = localtime()[0:5]
current_date = "{0}{1}{2}{3}{4}".format(current_date[0], current_date[1],
current_date[2], current_date[3],
current_date[4])
dest_files_in_container = models.OutputFileBlobContainerDestination(
container_url=
f"{config_resources.blob_container['url']}{config_resources.blob_container['sas_token']}",
path=f"{name_job}/{name_task}/{current_date}")
dest_files = models.OutputFileDestination(
container=dest_files_in_container)
trigger_upload = models.OutputFileUploadOptions(
upload_condition='taskCompletion')
upload_files = models.OutputFile(file_pattern='$AZ_BATCH_TASK_DIR/*',
destination=dest_files,
upload_options=trigger_upload)
outputs = []
outputs.append(upload_files)
tache = models.TaskAddParameter(id=name_task,
command_line=cmd,
multi_instance_settings=param_multi_inst,
resource_files=None,
environment_settings=None,
output_files=outputs)
batch_client.task.add(name_job, tache)
def add_tasks(batch_service_client, job_id, input_files, output_container_sas_url):
"""
Adds a task for each input file in the collection to the specified job.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str job_id: The ID of the job to which to add the tasks.
:param list input_files: A collection of input files. One task will be
created for each input file.
:param output_container_sas_token: A SAS token granting write access to
the specified Azure Blob storage container.
"""
print('Adding {} tasks to job [{}]...'.format(len(input_files), job_id))
tasks = list()
for idx, input_file in enumerate(input_files):
input_file_path=input_file.file_path
output_file_path="".join((input_file_path).split('.')[:-1]) + '.mp3'
command = "/bin/bash -c \"ffmpeg -i {} {} \"".format(input_file_path, output_file_path)
tasks.append(batch.models.TaskAddParameter(
id='Task{}'.format(idx),
command_line=command,
resource_files=[input_file],
output_files=[batchmodels.OutputFile(
file_pattern=output_file_path,
destination=batchmodels.OutputFileDestination(
container=batchmodels.OutputFileBlobContainerDestination(
container_url=output_container_sas_url)),
upload_options=batchmodels.OutputFileUploadOptions(
upload_condition=batchmodels.OutputFileUploadCondition.task_success))]
)
)
batch_service_client.task.add_collection(job_id, tasks)
def build_output_file(self, i):
"""
Uploads a local file to an Azure Blob storage container.
:rtype: `azure.batch.models.ResourceFile`
:return: A ResourceFile initialized with a SAS URL appropriate for Batch
tasks.
"""
# where to store the outputs
container_dest = models.OutputFileBlobContainerDestination(
container_url=self.CONTAINER_SAS_URL,
path=OUTPUT_FILE_PATTERN.format(i),
)
dest = models.OutputFileDestination(container=container_dest)
# under what conditions should you attempt to extract the outputs?
upload_options = models.OutputFileUploadOptions(
upload_condition=models.OutputFileUploadCondition.task_success)
# https://docs.microsoft.com/en-us/azure/batch/batch-task-output-files#specify-output-files-for-task-output
return models.OutputFile(
file_pattern=_CONTAINER_OUTPUT_FILE,
destination=dest,
upload_options=upload_options,
)
def _create_output_file(output_container_sas_url, output_file_path):
return batchmodels.OutputFile(
output_file_path,
destination=batchmodels.OutputFileDestination(
container=batchmodels.OutputFileBlobContainerDestination(
output_container_sas_url)),
upload_options=batchmodels.OutputFileUploadOptions(
batchmodels.OutputFileUploadCondition.task_success))
def add_tasks(batch_service_client, job_id, input_files,
output_container_sas_url):
"""
Adds a task for each input file in the collection to the specified job.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str job_id: The ID of the job to which to add the tasks.
:param list input_files: A collection of input files. One task will be created for each input file.
:param output_container_sas_url: A SAS token granting write access to the specified Azure Blob storage container.
"""
print('Adding {} tasks to job [{}]...'.format(
len(input_files) - 1, job_id))
tasks = []
output_pattern = "*-[01w]*"
task_idx = 0
for idx, input_file in enumerate(input_files):
input_file_path = input_file.file_path
subjid = "".join(input_file_path.split('.')[:-2])
subjid = "".join(subjid.split('-')[0])
node_file = [s for s in input_files if subjid in s]
node_file.append(s for s in input_files if "template" in s)
if subjid != "template":
command = "/bin/bash -c \"$ANTSPATH/antsRegistration -d 3 -o [{}-,{}-warped.nii.gz] " \
"-n Linear -w [0.005,0.995] -u 0 -r [template-aging1.nii.gz,{}-masked.nii.gz,1]" \
" -t Rigid[0.1] -m MI[template-aging1.nii.gz,{}-masked.nii.gz,1,32,Regular,0.25]" \
" -c [1000x500x250x100,1e-6,10] -f 8x4x2x1 -s 3x2x1x0vox" \
" -t Affine[0.1] -m MI[template-aging1.nii.gz,{}-masked.nii.gz,1,32,Regular,0.25]" \
" -c [1000x500x250x100,1e-6,10] -f 8x4x2x1 -s 3x2x1x0vox" \
" -t SyN[0.25] -m CC[template-aging1.nii.gz,{}-masked.nii.gz,1,4]" \
" -c [100x70x50x20,1e-6,10] -f 8x4x2x1 -s 3x2x1x0vox" \
"\"".format(subjid, subjid, subjid, subjid, subjid, subjid)
tasks.append(
batch.models.TaskAddParameter(
id='Task{}'.format(task_idx),
command_line=command,
environment_settings=[
batchmodels.EnvironmentSetting(
"ANTSPATH", "/usr/local/ants/v2.3.1/bin")
],
resource_files=node_file,
output_files=[
batchmodels.OutputFile(
output_pattern,
destination=batchmodels.OutputFileDestination(
container=batchmodels.
OutputFileBlobContainerDestination(
output_container_sas_url)),
upload_options=batchmodels.OutputFileUploadOptions(
batchmodels.OutputFileUploadCondition.
task_success))
]))
task_idx = task_idx + 1
batch_service_client.task.add_collection(job_id, tasks)
def add_tasks(batch_service_client, job_id, input_files,
output_container_sas_url):
"""
Adds a task for each input file in the collection to the specified job.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str job_id: The ID of the job to which to add the tasks.
:param list input_files: A collection of input files. One task will be
created for each input file.
:param output_container_sas_token: A SAS token granting write access to
the specified Azure Blob storage container.
"""
#print('Adding {} tasks to job [{}]...'.format(len(input_files), job_id))
print('Adding {} tasks to job [{}]...'.format(
_LOW_PRIORITY_POOL_NODE_COUNT, job_id))
tasks = list()
#for idx, input_file in enumerate(input_files):
for idx in range(_LOW_PRIORITY_POOL_NODE_COUNT):
# output_file_path='version{}.txt'.format(idx)
# command = "/bin/bash -c \"apt-get install git; \
# git clone https://github.com/uiuc-arc/probfuzz.git; \
# sudo apt-get install -y default-jdk;\
# cd ./probfuzz; \
# ./install.sh; \
# command = "/bin/bash -c \"git clone https://github.com/uiuc-arc/probfuzz.git; \
# cd ./probfuzz;\
# (cd language/antlr/ && wget http://www.antlr.org/download/antlr-4.7.1-complete.jar && ./run.sh); \
# ./check.py; \
# ./probfuzz.py 5\""
command = "/bin/bash -c \"cp -r /mnt/batch/tasks/startup/wd/probfuzz ./; \
cd probfuzz; \
./probfuzz.py 5\""
#command = "/bin/bash -c \"touch probfuzz.txt\""
tasks.append(
batch.models.TaskAddParameter(
id='Task{}'.format(idx),
command_line=command,
user_identity=batchmodels.UserIdentity(user_name="admin"),
#resource_files=input_files,
output_files=[
batchmodels.OutputFile(
file_pattern="./probfuzz/output/*",
destination=batchmodels.OutputFileDestination(
container=batchmodels.
OutputFileBlobContainerDestination(
container_url=output_container_sas_url)),
upload_options=batchmodels.OutputFileUploadOptions(
upload_condition=batchmodels.
OutputFileUploadCondition.task_success))
]))
batch_service_client.task.add_collection(job_id, tasks)
def create_output_file(filename, blobname):
container = batchmodels.OutputFileBlobContainerDestination(
container_url=output_sas_url, path=blobname)
destination = batchmodels.OutputFileDestination(
container=container)
upload_options = batchmodels.OutputFileUploadOptions(
upload_condition='taskCompletion')
return batchmodels.OutputFile(file_pattern=filename,
destination=destination,
upload_options=upload_options)
def add_tasks(batch_service_client, job_id, input_files,
output_container_sas_url):
"""
Adds a task for each input file in the collection to the specified job.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str job_id: The ID of the job to which to add the tasks.
:param list input_files: A collection of input files. One task will be
created for each input file.
:param output_container_sas_token: A SAS token granting write access to
the specified Azure Blob storage container.
"""
#print('Adding {} tasks to job [{}]...'.format(len(input_files), job_id))
print('Adding {} tasks to job [{}]...'.format(
_LOW_PRIORITY_POOL_NODE_COUNT + _DEDICATED_POOL_NODE_COUNT, job_id))
tasks = list()
for idx, input_file in enumerate(input_files):
if idx >= 8:
break
input_file_path = input_file.file_path
input_file_name = input_file_path.split('/')[-1].split('.')[0]
#output_file_path="".join((input_file_path).split('.')[:-2]) + 'metrics_out_0319' + '.txt'
# TODO: cp task to stan dir; unzip; build; change mnt script; upload metrics & summary to storage
command = "/bin/bash -c \" \
time ./tar_to_metrics.sh {0} &> {1}_log.txt; \
\"".format(input_file_path, input_file_name)
#
#./run_metrics.sh ./metrics.py /mnt/batch/tasks/workitems/RunByIterJobAll/job-1/Task*/wd/example-models/; \
#./runner_10.sh /mnt/batch/tasks/workitems/RunByIterJobAll/job-1/Task*/wd/example-models/ _*.csv; \
#pip2 install --upgrade pip2; \
#sudo apt-get install -y r-base-core; \
#command = "/bin/bash -c \"touch probfuzz.txt\""
tasks.append(
batch.models.TaskAddParameter(
id='Task_{}'.format(input_file_name),
command_line=command,
user_identity=batchmodels.UserIdentity(user_name="admin"),
resource_files=input_files,
output_files=[
batchmodels.OutputFile(
file_pattern="*.txt".format(idx),
destination=batchmodels.OutputFileDestination(
container=batchmodels.
OutputFileBlobContainerDestination(
container_url=output_container_sas_url)),
upload_options=batchmodels.OutputFileUploadOptions(
upload_condition=batchmodels.
OutputFileUploadCondition.task_success))
]))
batch_service_client.task.add_collection(job_id, tasks)
def add_task(
batch_service_client, job_id, task_id, num_instances,
application_cmdline, input_files, elevation_level,
output_file_names, output_container_sas,
coordination_cmdline, common_files):
"""
Adds a task for each input file in the collection to the specified job.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str job_id: The ID of the job to which to add the task.
:param str task_id: The ID of the task to be added.
:param str application_cmdline: The application commandline for the task.
:param list input_files: A collection of input files.
:param elevation_level: Elevation level used to run the task; either
'admin' or 'nonadmin'.
:type elevation_level: `azure.batch.models.ElevationLevel`
:param int num_instances: Number of instances for the task
:param str coordination_cmdline: The application commandline for the task.
:param list common_files: A collection of common input files.
"""
print('Adding {} task to job [{}]...'.format(task_id, job_id))
multi_instance_settings = None
if coordination_cmdline or (num_instances and num_instances > 1):
multi_instance_settings = batchmodels.MultiInstanceSettings(
number_of_instances=num_instances,
coordination_command_line=coordination_cmdline,
common_resource_files=common_files)
user = batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=elevation_level)
output_file = batchmodels.OutputFile(
file_pattern=output_file_names,
destination=batchmodels.OutputFileDestination(
container=batchmodels.OutputFileBlobContainerDestination(
container_url=output_container_sas)),
upload_options=batchmodels.OutputFileUploadOptions(
upload_condition=batchmodels.
OutputFileUploadCondition.task_completion))
task = batchmodels.TaskAddParameter(
id=task_id,
command_line=application_cmdline,
user_identity=batchmodels.UserIdentity(auto_user=user),
resource_files=input_files,
multi_instance_settings=multi_instance_settings,
output_files=[output_file])
batch_service_client.task.add(job_id, task)
def create_processing_tasks(block_blob_client, job_id, container_settings):
tasks = []
resource_files = []
container_url = azure_helpers.create_container_sas(
block_blob_client,
_CONTAINER_NAME,
datetime.datetime.utcnow() + datetime.timedelta(hours=24)
)
for task_script in ["petal_width.py", "sepal_width.py"]:
task_name = task_script.replace("_", "")
task_path = Path("resources", task_script)
script_stem = task_script.rstrip(".py")
# Shared access signature, allows access to a resource using a token for a specified amount of time
sas_url = azure_helpers.upload_blob_and_create_sas(
block_blob_client,
_CONTAINER_NAME,
task_name,
task_path,
datetime.datetime.utcnow() + datetime.timedelta(hours=1))
resource_file = batchmodels.ResourceFile(
file_path=task_name,
http_url=sas_url)
resource_files.append(resource_file)
# output data to upload to storage container
output_file = batchmodels.OutputFile(
file_pattern=f'output/{script_stem}.csv',
destination=batchmodels.OutputFileDestination(
container=batchmodels.OutputFileBlobContainerDestination(
container_url=container_url,
path=f'{job_id}/output/{script_stem}.csv'
)
),
upload_options=batchmodels.OutputFileUploadOptions(
upload_condition=batchmodels.OutputFileUploadCondition.task_completion
)
)
# add tasks with resources and outputs
tasks.append(batchmodels.TaskAddParameter(
id=script_stem,
command_line=f'python3 {task_name}',
resource_files=[resource_file],
container_settings=container_settings,
output_files=[output_file]
))
return resource_files, tasks
def add_tasks(batch_service_client, job_id, input_files,
output_container_sas_url):
"""Adds a task for each pair with input file and command line in the collection to the specified job.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str job_id: The ID of the job to which to add the tasks.
:param list input_files: A collection of input files.
:param output_container_sas_url: A SAS token granting write access to
the specified Azure Blob storage container."""
print('Adding [{}] tasks to job [{}]'.format(_MAX_TASKS, job_id))
tasks = list()
mcr = "".join(input_files[-4].file_path)
program = "".join(input_files[-3].file_path)
data = "".join(input_files[-2].file_path)
design = "".join(input_files[-1].file_path)
output_file_pattern = '*.mat'
cogns = ['cogn_ps', 'cogn_po']
for idx in range(1, _MAX_TASKS + 1):
command = "/bin/bash -c 'sudo apt-get update -qq && sudo apt-get install libxt6 default-jre -y;tar xf {};" \
"mcr/install -destinationFolder /tmp/mcr18b -mode silent -agreeToLicense yes;rm -rf mcr {} /tmp/ma*" \
";./{} /tmp/mcr18b/v95 {} {} {} {} {} 15'".format(
mcr, mcr, program, data, design, cogns[1], 'batch' + str(idx) + '.mat', idx)
tasks.append(
batchmodels.TaskAddParameter(
id='Task{}'.format(idx),
command_line=command,
resource_files=input_files,
output_files=[
batchmodels.OutputFile(
file_pattern=output_file_pattern,
destination=batchmodels.OutputFileDestination(
container=batchmodels.
OutputFileBlobContainerDestination(
container_url=output_container_sas_url)),
upload_options=batchmodels.OutputFileUploadOptions(
upload_condition='taskCompletion'))
],
user_identity=batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin))))
batch_service_client.task.add_collection(job_id, tasks)
def _generate_batch_output_file_spec(
storage_settings, container, saskey, seperator, condition, local_path,
include):
# type: (settings.StorageCredentialsSettings, str, str, str, str, str,
# str) -> batchmodels.OutputFile
"""Generate Batch output file spec with given local path and filter
:param settings.StorageCredentialsSettings storage_settings:
storage settings
:param str container: container
:param str saskey: sas key
:param str separator: dir separator
:param str condition: upload condition
:param str local_path: task local path
:param str include: include filter
:rtype: batchmodels.OutputFile
:return: Batch output file spec
"""
# set file pattern
if local_path.endswith(seperator):
fp = ''.join((local_path, include))
else:
fp = seperator.join((local_path, include))
# set upload condition
if condition == 'taskcompletion':
buc = batchmodels.OutputFileUploadCondition.task_completion
elif condition == 'taskfailure':
buc = batchmodels.OutputFileUploadCondition.task_failure
elif condition == 'tasksuccess':
buc = batchmodels.OutputFileUploadCondition.task_success
# generate spec
outfile = batchmodels.OutputFile(
file_pattern=fp,
destination=batchmodels.OutputFileDestination(
container=batchmodels.OutputFileBlobContainerDestination(
path='',
container_url='{}?{}'.format(
storage.generate_blob_container_uri(
storage_settings, container),
saskey)
)
),
upload_options=batchmodels.OutputFileUploadOptions(
upload_condition=buc
),
)
return outfile
def add_tasks(batch_service_client: batch.BatchServiceClient, job_id: str,
source_files: List[batchmodels.ResourceFile],
input_files: List[batchmodels.ResourceFile],
output_container_sas_url: str) -> None:
"""
Adds a task for each input file in the collection to the specified job.
:param batch_service_client: A Batch service client.
:param job_id: The ID of the job to which to add the tasks.
:param source_files: A collection of source files.
:param input_files: A collection of input files. One task will be created
for each input file.
:param output_container_sas_url: A SAS URL granting the specified
permissions to the output container.
"""
print('Adding {} tasks to job [{}]...'.format(len(input_files), job_id))
tasks = list()
for idx, input_file in enumerate(input_files):
input_file_path = input_file.file_path
output_file_path = "".join(
(os.path.basename(input_file_path)).split('.')[:-1]) + 'output.txt'
command = "/bin/bash -c \""\
"python3.7 -m pip install --upgrade pip && "\
"python3.7 -m pip install wheel && "\
f"python3.7 -m pip install -r {config._JOB_SCRIPT_PATH}/requirements.txt && "\
f"python3.7 {config._TASK_ENTRY_SCRIPT} {input_file_path} {output_file_path}"\
"\""
tasks.append(
batch.models.TaskAddParameter(
id='Task{}'.format(idx),
command_line=command,
resource_files=source_files + [input_file],
output_files=[
batchmodels.OutputFile(
file_pattern=output_file_path,
destination=batchmodels.OutputFileDestination(
container=batchmodels.
OutputFileBlobContainerDestination(
container_url=output_container_sas_url)),
upload_options=batchmodels.OutputFileUploadOptions(
upload_condition=batchmodels.
OutputFileUploadCondition.task_success))
]))
batch_service_client.task.add_collection(job_id, tasks)
def add_tasks(batch_service_client, job_id, packs, output_container_sas_url):
"""
Adds a task for each input file in the collection to the specified job.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str job_id: The ID of the job to which to add the tasks.
:param list packs: A collection of input files. One task will be
created for each input file.
:param output_container_sas_token: A SAS token granting write access to
the specified Azure Blob storage container.
"""
print('Adding {} tasks to job [{}]...'.format(len(packs), job_id))
tasks = list()
for idx, pack in enumerate(packs):
script = pack[0]
s = pack[1]
d = pack[2]
output_file_path = f"./stats_{d}_{s}.csv"
filename_path = f"{s}/{d}/stats/stats_{d}_{s}.csv"
command = "/bin/bash -c \"python3 {} --outstorage '{}' --outcontainer '{}' --sas '{}' --acckey '{}' --seckey '{}' --state '{}' --date '{}' --core '{}' --ram '{}'\"".format(
script.file_path,STORAGE_ACCOUNT_NAME,"testdelta",SAS_TOKEN,CUEBIQ_ACCESS_KEY,CUEBIQ_SECRET_KEY,s,d,CORE,RAM)
tasks.append(
batch.models.TaskAddParameter(
id='Task{}'.format(idx),
command_line=command,
resource_files=[script],
output_files=[
batchmodels.OutputFile(
file_pattern=output_file_path,
destination=batchmodels.OutputFileDestination(
container=batchmodels.OutputFileBlobContainerDestination(
container_url=output_container_sas_url,
path=filename_path)),
upload_options=batchmodels.OutputFileUploadOptions(
upload_condition=batchmodels.OutputFileUploadCondition.task_success))]
)
)
batch_service_client.task.add_collection(job_id, tasks)
def add_tasks(batch_service_client, job_id, input_files,
output_container_sas_url):
"""
Adds a task for each input file in the collection to the specified job.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str job_id: The ID of the job to which to add the tasks.
:param list input_files: A collection of input files. One task will be
created for each input file.
:param output_container_sas_token: A SAS token granting write access to
the specified Azure Blob storage container.
"""
print('Adding {} tasks to job [{}]...'.format(int(len(input_files) / 2),
job_id))
tasks = list()
for idx in range(int(len(input_files) / 2)):
input_file_path1 = input_files[2 * idx].file_path
input_file_path2 = input_files[2 * idx + 1].file_path
output_folder_path = input_file_path1.split('/')[1].split('_')[0]
command = "/bin/bash -c \"/mnt/batch/tasks/startup/wd/SPAdes-3.14.0-Linux/bin/spades.py -1 {} -2 {} -t 64 -m 250 --plasmid --meta -o {} \"".format(
input_file_path1, input_file_path2, output_folder_path)
tasks.append(
batch.models.TaskAddParameter(
id='Task{}'.format(idx),
command_line=command,
resource_files=[
input_files[2 * idx], input_files[2 * idx + 1]
],
output_files=[
batchmodels.OutputFile(
file_pattern=output_folder_path,
destination=batchmodels.OutputFileDestination(
container=batchmodels.
OutputFileBlobContainerDestination(
container_url=output_container_sas_url)),
upload_options=batchmodels.OutputFileUploadOptions(
upload_condition=batchmodels.
OutputFileUploadCondition.task_success))
]))
batch_service_client.task.add_collection(job_id, tasks)
def add_tasks(batch_service_client, job_id, input_files,
output_container_sas_url):
"""Adds a task for each pair with input file and command line in the collection to the specified job.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str job_id: The ID of the job to which to add the tasks.
:param list input_files: A collection of input files. One task will be
created for each input file
:param output_container_sas_url: A SAS token granting write access to
the specified Azure Blob storage container."""
print('Adding [{}] tasks to job [{}]'.format(len(input_files) - 2, job_id))
tasks = list()
output_file_pattern = '*-[01w]*'
template = "".join(input_files[-1].file_path.split('.')[:-2])
script = "".join(input_files[-2].file_path)
for idx, input_file in enumerate(input_files[:-2]):
input_file_path = input_file.file_path
subjid = "".join(input_file_path.split('.')[:-2])
command = "/bin/bash -c 'source /usr/local/software/addpaths;./{} {} {}'".format(
script, template, subjid)
tasks.append(
batchmodels.TaskAddParameter(
id='Task{}'.format(idx + 1),
command_line=command,
resource_files=[input_file, input_files[-2], input_files[-1]],
output_files=[
batchmodels.OutputFile(
file_pattern=output_file_pattern,
destination=batchmodels.OutputFileDestination(
container=batchmodels.
OutputFileBlobContainerDestination(
container_url=output_container_sas_url)),
upload_options=batchmodels.OutputFileUploadOptions(
upload_condition='taskCompletion'))
],
user_identity=batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin))))
batch_service_client.task.add_collection(job_id, tasks)
def add_tasks(batch_service_client, job_id, input_files,
output_container_sas_url, num_tasks):
"""Adds a task for each pair with input file and command line in the collection to the specified job.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str job_id: The ID of the job to which to add the tasks.
:param list input_files: A collection of input files.
:param output_container_sas_url: A SAS token granting write access to
the specified Azure Blob storage container.
:param num_tasks: Total number of tasks."""
print('Adding [{}] tasks to job [{}]'.format(num_tasks, job_id))
tasks = list()
output_file_pattern = '[ts]*'
for idx in range(num_tasks):
# input_file_path = input_file.file_path
# subjid = "".join(input_file_path.split('.')[:-2])
command = "/bin/bash -c 'source /usr/local/software/addpaths;" \
"randomise -i all_FA_skeletonised -m mean_FA_skeleton_mask -d design.mat -t design.con " \
"-D --T2 --uncorrp --seed={} -n {} -o {}_SEED{};cp -p ../stdout.txt stdout{}.txt'" \
"".format(idx+1, int(_RANDOMISE_ITER/_NUM_TASKS), _OUTPUT_PREFIX, idx+1, idx+1)
tasks.append(
batchmodels.TaskAddParameter(
id='Task{}'.format(idx + 1),
command_line=command,
resource_files=input_files,
output_files=[
batchmodels.OutputFile(
file_pattern=output_file_pattern,
destination=batchmodels.OutputFileDestination(
container=batchmodels.
OutputFileBlobContainerDestination(
container_url=output_container_sas_url)),
upload_options=batchmodels.OutputFileUploadOptions(
upload_condition='taskCompletion'))
],
user_identity=batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin))))
batch_service_client.task.add_collection(job_id, tasks)
def build_output_file(self, output_file,
container_path) -> azure.batch.models.ResourceFile:
"""
Uploads a local file to an Azure Blob storage container.
Args:
output_file: the name of the file produced as the output by the task
container_path: the name of the file in the container
Returns:
A ResourceFile initialized with a SAS URL appropriate for Batch tasks.
"""
# where to store the outputs
container_sas_url = (
self.container_client.url + "?" + generate_container_sas(
self.container_client.account_name,
self.container_client.container_name,
permission=ContainerSasPermissions(
read=True, write=True, delete=True, list=True),
expiry=datetime.datetime.utcnow() + datetime.timedelta(
hours=self.config.STORAGE_ACCESS_DURATION_HRS),
account_key=self.config.STORAGE_ACCOUNT_KEY,
))
destination = models.OutputFileDestination(
container=models.OutputFileBlobContainerDestination(
container_url=container_sas_url, path=container_path))
# Under what conditions should Azure Batch attempt to extract the outputs?
upload_options = models.OutputFileUploadOptions(
upload_condition=models.OutputFileUploadCondition.task_success)
# https://docs.microsoft.com/en-us/azure/batch/batch-task-output-files#specify-output-files-for-task-output
out = models.OutputFile(
file_pattern=output_file,
destination=destination,
upload_options=upload_options,
)
self.output_files.append(container_path)
return out
def add_tasks(batch_service_client, job_id, input_files, source_files, output_container_sas_url):
"""
Adds a task for each input file in the collection to the specified job.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str job_id: The ID of the job to which to add the tasks.
:param list input_files: A collection of input files. One task will be
created for each input file.
:param list source_files: A collection of source files.
:param output_container_sas_token: A SAS token granting write access to
the specified Azure Blob storage container.
"""
print('Adding {} tasks to job [{}]...'.format(len(input_files), job_id))
tasks = list()
for idx, input_file in enumerate(input_files):
input_file_path = input_file.file_path
output_file_path = "".join((input_file_path).split('.')[-2]) + '.output'
command = "/bin/bash -c \""\
"python3.7 -m pip install --upgrade pip && "\
f"python3.7 -m pip install -r {config._APP_PATH}/requirements.txt && "\
f"python3.7 {config._APP_MAIN_SCRIPT} {input_file_path} {output_file_path}"\
"\""
tasks.append(batch.models.TaskAddParameter(
id='Task{}'.format(idx),
command_line=command,
resource_files=source_files + [input_file],
output_files=[batchmodels.OutputFile(
file_pattern=output_file_path,
destination=batchmodels.OutputFileDestination(
container=batchmodels.OutputFileBlobContainerDestination(
container_url=output_container_sas_url)),
upload_options=batchmodels.OutputFileUploadOptions(
upload_condition=batchmodels.OutputFileUploadCondition.task_success))]
)
)
batch_service_client.task.add_collection(job_id, tasks)
def build_output_file(
file_pattern,
container_url,
file_path,
upload_condition=batchmodels.OutputFileUploadCondition.task_completion
):
"""Generates an output file object
:param str file_pattern: A pattern indicating which file(s) to upload.
:param str container_url: azure contaiuner URL with sas token.
:param str file_path: The destination for the output file(s)
:param str upload_condition: The conditions under which the task output file or set of files should be uploaded.
The default is taskcompletion. Possible values include: 'taskSuccess', 'taskFailure', 'taskCompletion'
:rtype: batchmodels.OutputFile
"""
output_file = batchmodels.OutputFile(
file_pattern=file_pattern,
destination=batchmodels.OutputFileDestination(
container=batchmodels.OutputFileBlobContainerDestination(
path=file_path, container_url=container_url)),
upload_options=batchmodels.OutputFileUploadOptions(
upload_condition=upload_condition))
return output_file
def submit_job_and_add_task(batch_client, block_blob_client, job_id, pool_id,
block_indices, input_files,
output_container_sas_url):
"""Submits a job to the Azure Batch service and adds
a task that runs a python script.
:param batch_client: The batch client to use.
:type batch_client: `batchserviceclient.BatchServiceClient`
:param block_blob_client: The storage block blob client to use.
:type block_blob_client: `azure.storage.blob.BlockBlobService`
:param str job_id: The id of the job to create.
:param str pool_id: The id of the pool to use.
"""
print(block_indices)
job = batchmodels.JobAddParameter(
id=job_id, pool_info=batchmodels.PoolInformation(pool_id=pool_id))
batch_client.job.add(job)
block_blob_client.create_container(_CONTAINER_NAME, fail_on_exist=False)
sas_url = common.helpers.upload_blob_and_create_sas(
block_blob_client, _CONTAINER_NAME, _SIMPLE_TASK_NAME,
_SIMPLE_TASK_PATH, _EXPIRY_TIME)
tasks = list()
# Count how many items are stored in the batch
inBatch = 0
for block, input_file in zip(block_indices, input_files):
input_file_path = input_file.file_path
output_file_path = "".join(
(input_file_path).split('.')[:-1]) + '_model.dat'
task_file = batchmodels.ResourceFile(file_path=_SIMPLE_TASK_NAME,
http_url=sas_url)
print(type(input_file), type(task_file))
tasks.append(
batchmodels.TaskAddParameter(
id='Task{}'.format(block),
command_line="python3 %s -b %d" % (_SIMPLE_TASK_NAME, block),
resource_files=[task_file, input_file],
output_files=[
batchmodels.OutputFile(
file_pattern=output_file_path,
destination=batchmodels.OutputFileDestination(
container=batchmodels.
OutputFileBlobContainerDestination(
container_url=output_container_sas_url)),
upload_options=batchmodels.OutputFileUploadOptions(
upload_condition=batchmodels.
OutputFileUploadCondition.task_success))
]))
inBatch += 1
# We can only send batches with up to 100 records
if inBatch > 99:
batch_client.task.add_collection(job.id, tasks)
tasks = []
inBatch = 0
if inBatch > 0:
batch_client.task.add_collection(job.id, tasks)
def create_merge_task(frame, task_id, job_id, depend_start, depend_end):
"""
Azure Batch task that executes the ImageMagick `convert` command
line to combine all of the output tiles into a single output image.
This task uses the task dependency model to make sure it
doesn't execute before it's dependent tasks have completed. This way
we know all of the output image tiles will exist.
:param frame: Frame number of the scene that this merge task is
processing.
:type frame: int
:param task_id: Identifier of the task.
:type task_id: str
:param job_id: Unique identifier of the job. Job identifiers are unique
within a single Azure Batch account.
:type job_id: str
:param depend_start: First task id of the dependency sequence. If each
frame is split into 16 tiles, then every 17th task will be a merge task
and that merge task will be dependent on the preceeding 16 tasks.
tile tasks 1 - 16, then merge, then tiles 18 - 34, then merge, etc.
:type depend_start: int
:param depend_end: Final task id of the dependency sequence. Explanation
for param `depend_start` applies here as well.
:type depend_end: int
"""
x_tiles = int(os.environ["X_TILES"])
y_tiles = int(os.environ["X_TILES"])
output_sas = os.environ["OUTPUT_CONTAINER_SAS"]
working_dir = os.environ["AZ_BATCH_TASK_WORKING_DIR"]
output_format = os.environ["OUTPUT_FORMAT"]
print("working_dir: {}".format(working_dir))
# crop to border means we need to use montage to tile the images. false means
# we can use convert -flatten to layer the images with transparent backgrounds
# convert is faster but needs RGBA
crop = os.environ["CROP_TO_BORDER"].lower()
if crop == "true":
command_line = montage_command(frame, x_tiles, y_tiles, output_format)
else:
command_line = convert_command(frame, output_format)
print("merge task command line: {}".format(command_line))
return models.TaskAddParameter(
id=pad_number(task_id, PAD_LEN_ID),
display_name="frame: {} - merge task".format(frame),
command_line=os_specific_command_line(command_line),
constraints=models.TaskConstraints(max_task_retry_count=2),
environment_settings=[
models.EnvironmentSetting("X_TILES", str(x_tiles)),
models.EnvironmentSetting("Y_TILES", str(y_tiles))
],
depends_on=models.TaskDependencies(
task_ids=get_dependent_tasks(depend_start, depend_end)),
resource_files=get_resource_files(x_tiles, y_tiles, frame),
output_files=[
models.OutputFile(
file_pattern="../stdout.txt",
destination=models.OutputFileDestination(
container=models.OutputFileBlobContainerDestination(
container_url=output_sas,
path="{}/logs/frame-{}/merge.stdout.log".format(
job_id, pad_number(frame, PAD_LEN_FRAME)))),
upload_options=models.OutputFileUploadOptions(
models.OutputFileUploadCondition.task_completion)),
models.OutputFile(
file_pattern="../stderr.txt",
destination=models.OutputFileDestination(
container=models.OutputFileBlobContainerDestination(
container_url=output_sas,
path="{}/logs/frame-{}/merge.stderr.log".format(
job_id, pad_number(frame, PAD_LEN_FRAME)))),
upload_options=models.OutputFileUploadOptions(
models.OutputFileUploadCondition.task_completion)),
models.OutputFile(
file_pattern="frame_*",
destination=models.OutputFileDestination(
container=models.OutputFileBlobContainerDestination(
container_url=output_sas,
path="{}/outputs/final".format(job_id))),
upload_options=models.OutputFileUploadOptions(
models.OutputFileUploadCondition.task_success))
])
def create_task(frame, task_id, job_id, tile_num, current_x, current_y):
"""
Azure Batch task that renders the given tile. Run Blender from the command
line and pass in the job manager script and the blend file.
:param frame: Frame number of the scene that this merge task is
processing.
:type frame: int
:param task_id: Identifier of the task.
:type task_id: str
:param job_id: Unique identifier of the job. Job identifiers are unique
within a single Azure Batch account.
:type job_id: str
:param tile_num: Number of the current tile.
:type tile_num: int
:param current_x: X value of the current tile, used to generate the render
border.
:type current_x: int
:param current_y: Y value of the current tile, used to generate the render
border.
:type current_y: int
"""
blend_file = os.environ["BLEND_FILE"]
output_sas = os.environ["OUTPUT_CONTAINER_SAS"]
optionalParams = os.environ["OPTIONAL_PARAMS"]
command_line = blender_command(blend_file, optionalParams)
# only print this once
if task_id == 1:
print("tile task command line: {}".format(command_line))
return models.TaskAddParameter(
id=pad_number(task_id, PAD_LEN_ID),
display_name="frame: {}, tile: {}".format(frame, tile_num),
command_line=os_specific_command_line(command_line),
constraints=models.TaskConstraints(max_task_retry_count=2),
environment_settings=[
models.EnvironmentSetting("X_TILES", os.environ["X_TILES"]),
models.EnvironmentSetting("Y_TILES", os.environ["Y_TILES"]),
models.EnvironmentSetting("CROP_TO_BORDER",
os.environ["CROP_TO_BORDER"]),
models.EnvironmentSetting("OUTPUT_FORMAT",
os.environ["OUTPUT_FORMAT"]),
models.EnvironmentSetting("BLEND_FILE", os.environ["BLEND_FILE"]),
models.EnvironmentSetting("CURRENT_FRAME", str(frame)),
models.EnvironmentSetting("CURRENT_TILE", str(tile_num)),
models.EnvironmentSetting("CURRENT_X", str(current_x)),
models.EnvironmentSetting("CURRENT_Y", str(current_y))
],
resource_files=[
models.ResourceFile(
"https://raw.githubusercontent.com/Azure/BatchExplorer-data/master/ncj/blender/scripts/python-task-manager.py",
"scripts/python-task-manager.py")
],
output_files=[
models.OutputFile(
file_pattern="../stdout.txt",
destination=models.OutputFileDestination(
container=models.OutputFileBlobContainerDestination(
container_url=output_sas,
path="{}/logs/frame-{}/tile-{}.stdout.log".format(
job_id, pad_number(frame, PAD_LEN_FRAME),
pad_number(tile_num, PAD_LEN_TILE)))),
upload_options=models.OutputFileUploadOptions(
models.OutputFileUploadCondition.task_completion)),
models.OutputFile(
file_pattern="../stderr.txt",
destination=models.OutputFileDestination(
container=models.OutputFileBlobContainerDestination(
container_url=output_sas,
path="{}/logs/frame-{}/tile-{}.stderr.log".format(
job_id, pad_number(frame, PAD_LEN_FRAME),
pad_number(tile_num, PAD_LEN_TILE)))),
upload_options=models.OutputFileUploadOptions(
models.OutputFileUploadCondition.task_completion)),
models.OutputFile(
file_pattern="../fileuploaderr.txt",
destination=models.OutputFileDestination(
container=models.OutputFileBlobContainerDestination(
container_url=output_sas,
path="{}/logs/frame-{}/tile-{}.file_upload_stderr.log".
format(job_id, pad_number(frame, PAD_LEN_FRAME),
pad_number(tile_num, PAD_LEN_TILE)))),
upload_options=models.OutputFileUploadOptions(
models.OutputFileUploadCondition.task_completion)),
models.OutputFile(
file_pattern="../fileuploadout.txt",
destination=models.OutputFileDestination(
container=models.OutputFileBlobContainerDestination(
container_url=output_sas,
path="{}/logs/frame-{}/tile-{}.file_upload_stdout.log".
format(job_id, pad_number(frame, PAD_LEN_FRAME),
pad_number(tile_num, PAD_LEN_TILE)))),
upload_options=models.OutputFileUploadOptions(
models.OutputFileUploadCondition.task_completion)),
models.OutputFile(
file_pattern="tile_*",
destination=models.OutputFileDestination(
container=models.OutputFileBlobContainerDestination(
container_url=output_sas,
path="{}/outputs/frame-{}".format(
job_id, pad_number(frame, PAD_LEN_FRAME)))),
upload_options=models.OutputFileUploadOptions(
models.OutputFileUploadCondition.task_success))
])
def submit_job_and_add_task(batch_client, block_blob_client, job_id, pool_id,
storage_account_name):
"""Submits a job to the Azure Batch service and adds
a task that runs a python script.
:param batch_client: The batch client to use.
:type batch_client: `batchserviceclient.BatchServiceClient`
:param block_blob_client: The storage block blob client to use.
:type block_blob_client: `azure.storage.blob.BlockBlobService`
:param str job_id: The id of the job to create.
:param str pool_id: The id of the pool to use.
"""
job = batchmodels.JobAddParameter(
id=job_id, pool_info=batchmodels.PoolInformation(pool_id=pool_id))
batch_client.job.add(job)
output_container_sas = common.helpers.create_container_and_create_sas(
block_blob_client,
job_id,
azureblob.BlobPermissions.WRITE,
expiry=None,
timeout=120)
output_container_sas_url = 'https://{}.blob.core.windows.net/{}?{}'.format(
storage_account_name, job_id, output_container_sas)
app_file_list = get_resource_file_list_from_container(
block_blob_client, _APP_CONTAINER_NAME)
blob_list = block_blob_client.list_blobs(_RESOURCE_CONTAINER_NAME)
for blob in blob_list:
(blob_base_name, blob_extension) = os.path.splitext(blob.name)
output_file_name = f"{blob_base_name}_out{blob_extension}"
command_line = f"{_APP_EXE_NAME} {_APP_EXTRA_ARGS} {blob.name} {output_file_name}"
task_id = f"{_APP_EXE_NAME}_{blob_base_name}_Task"
resource_sas_url = common.helpers.create_sas_url(
block_blob_client, _RESOURCE_CONTAINER_NAME, blob.name,
azureblob.BlobPermissions.READ,
datetime.datetime.utcnow() + datetime.timedelta(hours=1))
resource_file = batchmodels.ResourceFile(file_path=blob.name,
http_url=resource_sas_url)
print(resource_sas_url)
print(app_file_list)
print(f"Creating task ({task_id}): " + command_line)
output_file = batchmodels.OutputFile(
file_pattern=output_file_name,
destination=batchmodels.OutputFileDestination(
container=batchmodels.OutputFileBlobContainerDestination(
container_url=output_container_sas_url)),
upload_options=batchmodels.OutputFileUploadOptions(
upload_condition=batchmodels.OutputFileUploadCondition.
task_completion))
task = batchmodels.TaskAddParameter(id=task_id,
command_line=command_line,
resource_files=app_file_list +
[resource_file],
output_files=[output_file])
batch_client.task.add(job_id=job.id, task=task)
batchmodels.ResourceFile(
file_path=analysis_grid_names[n],
blob_source=analysis_grid_sas_urls[n]),
batchmodels.ResourceFile(file_path="sky_mtx.json",
blob_source=sky_mtx_sas_url),
batchmodels.ResourceFile(file_path="surfaces.json",
blob_source=surfaces_sas_url)
],
output_files=[
batchmodels.OutputFile(
file_pattern="*_result.json",
destination=batchmodels.OutputFileDestination(
container=batchmodels.
OutputFileBlobContainerDestination(
container_url=container_sas_url)),
upload_options=batchmodels.OutputFileUploadOptions(
upload_condition="taskCompletion"))
],
user_identity=batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
elevation_level=batchmodels.ElevationLevel.admin,
scope=batchmodels.AutoUserScope.task)))
batch_client.task.add(job_id=job_id, task=task)
print("Task created: {0:} and assigned to {1:}".format(
task_id, job_id))
# Wait for tasks to complete
for job_id in job_ids:
common.helpers.wait_for_tasks_to_complete(batch_client, job_id,
datetime.timedelta(hours=4))
# # Get a list of all the tasks currently running and wait for them to all complete
# all_tasks = []
def create_task(dataset, command, dependencies, max_wall_clock, production):
if production:
container = RESULTS_CONTAINER
else:
container = TEST_RESULTS_CONTAINER + "/" + \
generate_task_name(dataset.name)
output_files = [
# Upload results
batch_models.OutputFile(
file_pattern="$AZ_BATCH_TASK_WORKING_DIR/results/**/*",
upload_options=batch_models.OutputFileUploadOptions(
upload_condition=batch_models.OutputFileUploadCondition.
task_success),
destination=batch_models.OutputFileDestination(
container=batch_models.OutputFileBlobContainerDestination(
path=dataset.data_dir, container_url=container +
SAS_TOKEN))),
batch_models.OutputFile(
file_pattern=
f"$AZ_BATCH_NODE_ROOT_DIR/fsmounts/{FILE_SHARE_NAME}/*.csv",
upload_options=batch_models.OutputFileUploadOptions(
upload_condition=batch_models.OutputFileUploadCondition.
task_success),
destination=batch_models.OutputFileDestination(
container=batch_models.OutputFileBlobContainerDestination(
container_url=container + SAS_TOKEN))),
batch_models.OutputFile(
file_pattern=
f"$AZ_BATCH_NODE_ROOT_DIR/fsmounts/{FILE_SHARE_NAME}/last-update/*",
upload_options=batch_models.OutputFileUploadOptions(
upload_condition=batch_models.OutputFileUploadCondition.
task_success),
destination=batch_models.OutputFileDestination(
container=batch_models.OutputFileBlobContainerDestination(
path="last-update", container_url=container + SAS_TOKEN))),
# Upload stderr and stdout
batch_models.OutputFile(
file_pattern="$AZ_BATCH_TASK_DIR/std*.txt",
upload_options=batch_models.OutputFileUploadOptions(
upload_condition=batch_models.OutputFileUploadCondition.
task_completion),
destination=batch_models.OutputFileDestination(
container=batch_models.OutputFileBlobContainerDestination(
path=DATETIME_NOWISH + "/" +
generate_task_name(dataset.name),
container_url=PROCESS_LOG_CONTAINER + "/" + SAS_TOKEN)))
]
return batch_models.TaskAddParameter(
id=generate_task_name(dataset.name),
display_name=(dataset.name + "_python_script_job"),
command_line=command,
resource_files=[
batch_models.ResourceFile(storage_container_url=CONFIG_CONTAINER +
SAS_TOKEN,
blob_prefix=dataset.name + CONFIG_FILE)
],
depends_on=batch_models.TaskDependencies(task_ids=dependencies),
user_identity=batch_models.UserIdentity(
auto_user=batch_models.AutoUserSpecification(
scope='pool', elevation_level='admin')),
container_settings=batch_models.TaskContainerSettings(
image_name=DOCKER_CONTAINER_URL,
container_run_options='-w /home/rstudio/covid-rt-estimates'),
constraints=batch_models.TaskConstraints(
max_wall_clock_time=datetime.timedelta(minutes=max_wall_clock)),
output_files=output_files)
def _process_storage_output_data(config, native, is_windows, output_data):
# type: (dict, bool, bool, dict) -> str
"""Process output data to egress to Azure storage
:param dict config: configuration dict
:param bool native: is native container pool
:param bool is_windows: is windows pool
:param dict output_data: config spec with output_data
:rtype: list
:return: OutputFiles or args to pass to blobxfer script
"""
# get gluster host/container paths and encryption settings
gluster_host, gluster_container = _get_gluster_paths(config)
encrypt = settings.batch_shipyard_encryption_enabled(config)
# parse storage output data blocks
args = []
for xfer in output_data:
storage_settings = settings.credentials_storage(
config, settings.data_storage_account_settings(xfer))
remote_path = settings.data_remote_path(xfer)
# derive container from remote_path
container = settings.data_container_from_remote_path(
xfer, rp=remote_path)
eo = settings.data_blobxfer_extra_options(xfer)
if native and util.is_not_empty(eo):
raise ValueError(
'native container pool does not support '
'blobxfer_extra_options')
# append appropriate option for fshare
if settings.data_is_file_share(xfer) and '--mode file' not in eo:
eo = '--mode file {}'.format(eo)
if '--mode file' in eo:
if native:
raise ValueError(
'native container pool does not support fileshares')
# create saskey for file share with rwdl perm
saskey = storage.create_file_share_saskey(
storage_settings, container, 'egress', create_share=True)
else:
# create saskey for container with rwdl perm
saskey = storage.create_blob_container_saskey(
storage_settings, container, 'egress', create_container=True)
includes = settings.data_include(xfer)
excludes = settings.data_exclude(xfer)
# convert include/excludes into extra options
filters = _convert_filter_to_blobxfer_option(includes, excludes)
local_path = settings.data_local_path(xfer, True, task_wd=False)
# auto replace container path for gluster with host path
if (util.is_not_empty(gluster_container) and
local_path.startswith(gluster_container)):
local_path = local_path.replace(gluster_container, gluster_host, 1)
if native:
if util.is_not_empty(excludes):
raise ValueError(
'native container pool does not support excludes')
if is_windows:
sep = '\\'
else:
sep = '/'
if util.is_none_or_empty(includes):
include = '**{}*'.format(sep)
if not local_path.endswith(sep):
fp = sep.join((local_path, include))
else:
fp = ''.join((local_path, include))
of = batchmodels.OutputFile(
file_pattern=fp,
destination=batchmodels.OutputFileDestination(
container=batchmodels.OutputFileBlobContainerDestination(
path='',
container_url='{}?{}'.format(
storage.generate_blob_container_uri(
storage_settings, container),
saskey)
)
),
upload_options=batchmodels.OutputFileUploadOptions(
upload_condition=batchmodels.
OutputFileUploadCondition.task_success
),
)
args.append(of)
else:
# construct argument
# kind:encrypted:<sa:ep:saskey:remote_path>:local_path:eo
creds = crypto.encrypt_string(
encrypt,
'{},{},{},{}'.format(
storage_settings.account, storage_settings.endpoint,
saskey, remote_path),
config)
args.append('"{bxver},e,{enc},{creds},{lp},{eo}"'.format(
bxver=_BLOBXFER_VERSION,
enc=encrypt,
creds=creds,
lp=local_path,
eo=' '.join((filters, eo)).lstrip(),
))
return args
def _generate_batch_output_file_spec(is_windows, separator, storage_settings,
saskey, container, remote_path, condition,
local_path, include):
# type: (bool, str, settings.StorageCredentialsSettings, str, str, str,
# str, str, str) -> batchmodels.OutputFile
"""Generate Batch output file spec with given local path and filter
:param bool is_windows: is windows pool
:param str separator: dir separator
:param settings.StorageCredentialsSettings storage_settings:
storage settings
:param str saskey: sas key
:param str container: container
:param str remote_path: remote path
:param str condition: upload condition
:param str local_path: task local path
:param str include: include filter
:rtype: batchmodels.OutputFile
:return: Batch output file spec
"""
# set file pattern
if local_path.endswith(separator):
fp = ''.join((local_path, include))
else:
fp = separator.join((local_path, include))
# set upload condition
if condition == 'taskcompletion':
buc = batchmodels.OutputFileUploadCondition.task_completion
elif condition == 'taskfailure':
buc = batchmodels.OutputFileUploadCondition.task_failure
elif condition == 'tasksuccess':
buc = batchmodels.OutputFileUploadCondition.task_success
# strip container from remote path
rp = remote_path.split('/')
if len(rp) > 1:
rp = rp[1:]
if '*' not in fp and '?' not in fp:
# limited resolution of file path/pattern
if is_windows:
tmp = fp.replace('%AZ_BATCH_TASK_DIR%\\',
'').replace('%AZ_BATCH_TASK_WORKING_DIR%\\',
'wd\\')
else:
tmp = fp.replace('$AZ_BATCH_TASK_DIR/',
'').replace('$AZ_BATCH_TASK_WORKING_DIR/',
'wd/')
rp.append(tmp)
rp = '/'.join(rp)
else:
rp = ''
# generate spec
outfile = batchmodels.OutputFile(
file_pattern=fp,
destination=batchmodels.OutputFileDestination(
container=batchmodels.OutputFileBlobContainerDestination(
path=rp,
container_url='{}?{}'.format(
storage.generate_blob_container_uri(
storage_settings, container), saskey))),
upload_options=batchmodels.OutputFileUploadOptions(
upload_condition=buc),
)
return outfile
def test_batch_tasks(self, batch_job, **kwargs):
client = self.create_sharedkey_client(**kwargs)
# Test Create Task with Auto Complete
exit_conditions = models.ExitConditions(
exit_codes=[models.ExitCodeMapping(1, models.ExitOptions(models.JobAction.terminate))],
exit_code_ranges=[models.ExitCodeRangeMapping(2, 4, models.ExitOptions(models.JobAction.disable))],
default=models.ExitOptions(models.JobAction.none))
task_param = models.TaskAddParameter(
id=self.get_resource_name('batch_task1_'),
command_line='cmd /c "echo hello world"',
exit_conditions=exit_conditions
)
try:
client.task.add(batch_job.id, task_param)
except models.BatchErrorException as e:
message = "{}: ".format(e.error.code, e.error.message)
for v in e.error.values:
message += "\n{}: {}".format(v.key, v.value)
raise Exception(message)
task = client.task.get(batch_job.id, task_param.id)
self.assertIsInstance(task, models.CloudTask)
self.assertEqual(task.exit_conditions.default.job_action, models.JobAction.none)
self.assertEqual(task.exit_conditions.exit_codes[0].code, 1)
self.assertEqual(task.exit_conditions.exit_codes[0].exit_options.job_action, models.JobAction.terminate)
# Test Create Task with Output Files
container_url = "https://test.blob.core.windows.net:443/test-container"
outputs = [
models.OutputFile(
file_pattern="../stdout.txt",
destination=models.OutputFileDestination(
container=models.OutputFileBlobContainerDestination(
container_url=container_url, path="taskLogs/output.txt")),
upload_options=models.OutputFileUploadOptions(
upload_condition=models.OutputFileUploadCondition.task_completion)),
models.OutputFile(
file_pattern="../stderr.txt",
destination=models.OutputFileDestination(
container=models.OutputFileBlobContainerDestination(
container_url=container_url, path="taskLogs/error.txt")),
upload_options=models.OutputFileUploadOptions(
upload_condition=models.OutputFileUploadCondition.task_failure)),
]
task_param = models.TaskAddParameter(
id=self.get_resource_name('batch_task2_'),
command_line='cmd /c "echo hello world"',
output_files=outputs
)
client.task.add(batch_job.id, task_param)
task = client.task.get(batch_job.id, task_param.id)
self.assertIsInstance(task, models.CloudTask)
self.assertEqual(len(task.output_files), 2)
# Test Create Task with Auto User
auto_user = models.AutoUserSpecification(
scope=models.AutoUserScope.task,
elevation_level=models.ElevationLevel.admin)
task_param = models.TaskAddParameter(
id=self.get_resource_name('batch_task3_'),
command_line='cmd /c "echo hello world"',
user_identity=models.UserIdentity(auto_user=auto_user)
)
client.task.add(batch_job.id, task_param)
task = client.task.get(batch_job.id, task_param.id)
self.assertIsInstance(task, models.CloudTask)
self.assertEqual(task.user_identity.auto_user.scope, models.AutoUserScope.task)
self.assertEqual(task.user_identity.auto_user.elevation_level, models.ElevationLevel.admin)
# Test Create Task with Token Settings
task_param = models.TaskAddParameter(
id=self.get_resource_name('batch_task4_'),
command_line='cmd /c "echo hello world"',
authentication_token_settings=models.AuthenticationTokenSettings(
access=[models.AccessScope.job])
)
client.task.add(batch_job.id, task_param)
task = client.task.get(batch_job.id, task_param.id)
self.assertIsInstance(task, models.CloudTask)
self.assertEqual(task.authentication_token_settings.access[0], models.AccessScope.job)
# Test Create Task with Container Settings
task_param = models.TaskAddParameter(
id=self.get_resource_name('batch_task5_'),
command_line='cmd /c "echo hello world"',
container_settings=models.TaskContainerSettings(
image_name='windows_container:latest',
registry=models.ContainerRegistry('username', 'password'))
)
client.task.add(batch_job.id, task_param)
task = client.task.get(batch_job.id, task_param.id)
self.assertIsInstance(task, models.CloudTask)
self.assertEqual(task.container_settings.image_name, 'windows_container:latest')
self.assertEqual(task.container_settings.registry.user_name, 'username')
# Test Create Task with Run-As-User
task_param = models.TaskAddParameter(
id=self.get_resource_name('batch_task6_'),
command_line='cmd /c "echo hello world"',
user_identity=models.UserIdentity(user_name='task-user')
)
client.task.add(batch_job.id, task_param)
task = client.task.get(batch_job.id, task_param.id)
self.assertIsInstance(task, models.CloudTask)
self.assertEqual(task.user_identity.user_name, 'task-user')
# Test Add Task Collection
tasks = []
for i in range(7, 10):
tasks.append(models.TaskAddParameter(
self.get_resource_name('batch_task{}_'.format(i)), 'cmd /c "echo hello world"'))
result = client.task.add_collection(batch_job.id, tasks)
self.assertIsInstance(result, models.TaskAddCollectionResult)
self.assertEqual(len(result.value), 3)
self.assertEqual(result.value[0].status, models.TaskAddStatus.success)
# Test List Tasks
tasks = list(client.task.list(batch_job.id))
self.assertEqual(len(tasks), 9)
# Test Count Tasks
task_counts = client.job.get_task_counts(batch_job.id)
self.assertIsInstance(task_counts, models.TaskCounts)
self.assertEqual(task_counts.completed, 0)
self.assertEqual(task_counts.succeeded, 0)
self.assertEqual(task_counts.validation_status, models.TaskCountValidationStatus.validated)
# Test Terminate Task
response = client.task.terminate(batch_job.id, task_param.id)
self.assertIsNone(response)
task = client.task.get(batch_job.id, task_param.id)
self.assertEqual(task.state, models.TaskState.completed)
# Test Reactivate Task
response = client.task.reactivate(batch_job.id, task_param.id)
self.assertIsNone(response)
task = client.task.get(batch_job.id, task_param.id)
self.assertEqual(task.state, models.TaskState.active)
# Test Update Task
response = client.task.update(
batch_job.id, task_param.id,
constraints=models.TaskConstraints(max_task_retry_count=1))
self.assertIsNone(response)
# Test Get Subtasks
# TODO: Test with actual subtasks
subtasks = client.task.list_subtasks(batch_job.id, task_param.id)
self.assertIsInstance(subtasks, models.CloudTaskListSubtasksResult)
self.assertEqual(subtasks.value, [])
# Test Delete Task
response = client.task.delete(batch_job.id, task_param.id)
self.assertIsNone(response)
def add_task(batch_service_client, job_id, input_files,
output_container_sas_url, task_name, input_file_dir):
"""
Adds a task for each input file in the collection to the specified job.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str job_id: The ID of the job to which to add the tasks.
:param list input_files: A collection of input files. One task will be
created for each input file.
:param output_container_sas_token: A SAS token granting write access to
the specified Azure Blob storage container.
"""
tasks = list()
# my own tasks section
logging.info('Adding dashcam task [{task}] to job [{job}]...'.format(
task=task_name, job=job_id))
output_file_path = "/mnt/batch/tasks/workitems/{jobname}/job-1/{taskname}/wd".format(
jobname=job_id, taskname=task_name)
logging.info('input_file_dir: {}'.format(input_file_dir))
logging.info('output_file_path: {}'.format(output_file_path))
# Command uses the script from https://github.com/ehendrix23/tesla_dashcam
command = '''tesla_dashcam --no-timestamp --monitor_trigger {input} --output {output} --motion_only --mirror --layout DIAMOND --fps 33 --quality HIGH --slowdown 5'''.format(
input=input_file_dir, output=output_file_path)
logging.info('command: {}'.format(command))
tasks.append(
batch.models.TaskAddParameter(
id=task_name,
command_line=command,
constraints=batchmodels.TaskConstraints(
max_wall_clock_time=datetime.timedelta(minutes=60),
max_task_retry_count=1),
resource_files=input_files,
output_files=[
batchmodels.OutputFile(
file_pattern='*.mp4',
destination=batchmodels.OutputFileDestination(
container=batchmodels.
OutputFileBlobContainerDestination(
path='',
container_url=
output_container_sas_url #os.environ["_SAS"]
)),
upload_options=batchmodels.OutputFileUploadOptions(
upload_condition=batchmodels.OutputFileUploadCondition.
task_success)),
batchmodels.OutputFile(
file_pattern='../std*.txt',
destination=batchmodels.OutputFileDestination(
container=batchmodels.
OutputFileBlobContainerDestination(
path=task_name,
container_url=
output_container_sas_url #os.environ["_SAS"]
)),
upload_options=batchmodels.OutputFileUploadOptions(
upload_condition=batchmodels.OutputFileUploadCondition.
task_success)),
batchmodels.OutputFile(
file_pattern='../fileupload*.txt',
destination=batchmodels.OutputFileDestination(
container=batchmodels.
OutputFileBlobContainerDestination(
path=task_name,
container_url=
output_container_sas_url #os.environ["_SAS"]
)),
upload_options=batchmodels.OutputFileUploadOptions(
upload_condition=batchmodels.OutputFileUploadCondition.
task_success))
]))
batch_service_client.task.add_collection(job_id, tasks)