def pool_create():
image_reference = batchmodel.ImageReference(
publisher=config_azure['batch_pool_image_publisher'],
offer=config_azure['batch_pool_image_offer'],
sku=config_azure['batch_pool_image_sku'])
vm_config = batchmodel.VirtualMachineConfiguration(
image_reference=image_reference,
node_agent_sku_id=config_azure['batch_pool_node_agent_sku'])
vm_start_task = batchmodel.StartTask(
command_line=
'/bin/bash -c "sudo yum -y install epel-release; sudo yum -y install python36 python36-devel python36-tools; sudo python36 -m ensurepip; sudo yum -y install openmpi openmpi-devel; sudo env MPICC=/usr/lib64/openmpi/bin/mpicc pip3 install mpi4py numpy; sudo pip3 --yes uninstall azure azure-common azure-storage; sudo pip3 install azure-storage azure-batch"',
user_identity=batchmodel.UserIdentity(
auto_user=batchmodel.AutoUserSpecification(
scope=batchmodel.AutoUserScope.pool,
elevation_level=batchmodel.ElevationLevel.admin)),
wait_for_success=True)
batch_service.pool.add(pool=batchmodel.PoolAddParameter(
id=config_azure['batch_pool_name'],
vm_size=config_azure['batch_pool_vm_size'],
virtual_machine_configuration=vm_config,
target_dedicated_nodes=config_azure[
'batch_pool_target_dedicated_nodes'],
enable_inter_node_communication=True,
start_task=vm_start_task),
raw=True)
def run_commands(batch_client, block_blob_client, job_id, pool_id):
"""Run the start commands listed in the file "start_commands" on
all the nodes of the Azure Batch service.
:param batch_client: The batch client to use.
:type batch_client: `batchserviceclient.BatchServiceClient`
:param block_blob_client: The storage block blob client to use.
:type block_blob_client: `azure.storage.blob.BlockBlobService`
:param str job_id: The id of the job to create.
:param str pool_id: The id of the pool to use.
"""
task_commands = get_list_from_file('configs/start_commands')
logging.info(task_commands)
user = batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)
start = time.time()
job = batchmodels.JobAddParameter(
id=job_id, pool_info=batchmodels.PoolInformation(pool_id=pool_id))
batch_client.job.add(job)
logging.info('job created in seconds {}'.format(time.time() - start))
start = time.time()
nodes = list(batch_client.compute_node.list(pool_id))
tasks = [batchmodels.TaskAddParameter(
id="EBOTask-{}".format(i),
command_line=common.helpers.wrap_commands_in_shell('linux', task_commands),
user_identity=batchmodels.UserIdentity(auto_user=user)) \
for i in xrange(len(nodes))]
batch_client.task.add_collection(job.id, tasks)
logging.info('task created in seconds {}'.format(time.time() - start))
def add_tasks(batch_service_client, job_id, task_id, number_to_test):
"""
Adds a task for each input file in the collection to the specified job.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str job_id: The ID of the job to which to add the tasks.
created for each input file.
:number_to_test: number you want to know if it's prime or not.
"""
print('Adding tasks to job [{}]...'.format(job_id))
# This is the user who run the command inside the container.
# An unprivileged one
user = batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.task,
elevation_level=batchmodels.ElevationLevel.non_admin)
# This is the docker image we want to run
task_container_settings = batchmodels.TaskContainerSettings(
image_name=config._DOCKER_IMAGE, container_run_options='--rm')
# The container needs this argument to be executed
task = batchmodels.TaskAddParameter(
id=task_id,
command_line='python /is_prime.py ' + str(number_to_test),
container_settings=task_container_settings,
user_identity=batchmodels.UserIdentity(auto_user=user))
batch_service_client.task.add(job_id, task)
def create_job(batch_service_client, job_id, resource_files, pool_id):
"""
Creates a job with the specified ID, associated with the specified pool.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str job_id: The ID for the job.
:param str pool_id: The ID for the pool.
"""
print('Creating job [{}]...'.format(job_id))
job = batch.models.JobAddParameter(
id=job_id,
pool_info=batch.models.PoolInformation(pool_id=pool_id),
job_preparation_task=batchmodels.JobPreparationTask(
command_line="/bin/bash -c \"\
sudo apt-get install -y --reinstall make; \
sudo ./installrpy.sh; \
tar -xf cmdstan.tar.gz; \
cd cmdstan; \
make build; \
cd -; \
if sudo test -d ../../cmdstan; then echo stanexist; exit 0; fi; \
sudo mv * ../../; \
\"",
resource_files=resource_files,
wait_for_success=True,
user_identity=batchmodels.UserIdentity(user_name="admin"),
))
#
batch_service_client.job.add(job)
def generate_task(spark_client, job, application_tasks):
resource_files = []
for application, task in application_tasks:
task_definition_resource_file = helpers.upload_text_to_container(
container_name=job.id,
application_name=application.name + ".yaml",
file_path=application.name + ".yaml",
content=yaml.dump(task),
blob_client=spark_client.blob_client,
)
resource_files.append(task_definition_resource_file)
task_cmd = __app_cmd()
# Create task
task = batch_models.JobManagerTask(
id=job.id,
command_line=helpers.wrap_commands_in_shell([task_cmd]),
resource_files=resource_files,
kill_job_on_completion=False,
allow_low_priority_node=True,
user_identity=batch_models.UserIdentity(
auto_user=batch_models.AutoUserSpecification(
scope=batch_models.AutoUserScope.task, elevation_level=batch_models.ElevationLevel.admin)),
)
return task
def create_job(batch_service_client, job_id, resource_files, pool_id):
"""
Creates a job with the specified ID, associated with the specified pool.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str job_id: The ID for the job.
:param str pool_id: The ID for the pool.
"""
print('Creating job [{}]...'.format(job_id))
job = batch.models.JobAddParameter(
id=job_id,
pool_info=batch.models.PoolInformation(pool_id=pool_id),
job_preparation_task=batchmodels.JobPreparationTask(
command_line="/bin/bash -c \"sudo apt-get -y update; \
sudo chmod o+w /usr/local/lib/R/site-library; \
./installrpy.sh; \
tar -xf cmdstan.tar.gz; \
cd cmdstan; \
make build; \
cd -; \
\"",
wait_for_success=True,
resource_files=resource_files,
user_identity=batchmodels.UserIdentity(user_name="admin"),
))
batch_service_client.job.add(job)
def add_tasks(batch_service_client, job_id, task_id):
"""
Adds a task for each input file in the collection to the specified job.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str job_id: The ID of the job to which to add the tasks.
created for each input file.
:number_to_test: number you want to know if it's prime or not.
"""
print('Adding tasks to job [{}]...'.format(job_id))
# This is the user who run the command inside the container.
# An unprivileged one
user = batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.task,
elevation_level=batchmodels.ElevationLevel.non_admin)
# This is the docker image we want to run
task_container_settings = batchmodels.TaskContainerSettings(
image_name=config._DOCKER_IMAGE,
container_run_options='--rm -e PYTHONUNBUFFERED=1')
# The container needs this argument to be executed
# remember we run the container like: docker ... imagename python /is_prime.py number
task = batchmodels.TaskAddParameter(
id=task_id,
command_line=
"python main.py quantity --job-id=2 --keyword='covid19' --question='Is covid19 man made' --num-papers=10",
container_settings=task_container_settings,
user_identity=batchmodels.UserIdentity(auto_user=user))
batch_service_client.task.add(job_id, task)
def add_training_task(batch_service_client, job_id, command, displayName, containerName, blobName):
"""
Adds a task to the specified job.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str jobId: The ID of the job to which to add the tasks.
:param command: commandLine activity to do
:param displayName display name of task
:param containerName name of container with training data
:param blobName name of training data blob
"""
# sample Linux command "/bin/bash -c \"ffmpeg -i {} {} \"
# sample Windows command "cmd /c cd .. & cd C:\Users\chbuja"
GUID = str(uuid.uuid4())[:63]
autouser = batchmodels.AutoUserSpecification(
scope='task', elevation_level='admin')
userId = batchmodels.UserIdentity(auto_user=autouser)
tasks = list()
tasks.append(batch.models.TaskAddParameter(
id='{}'.format(GUID),
command_line=command,
display_name=displayName,
user_identity=userId
))
batch_service_client.task.add_collection(job_id, tasks)
def create_pool(batch_client, pool_id, vm_size, vm_count, app_files):
"""Creates an Azure Batch pool with the specified id.
:param batch_client: The batch client to use.
:type batch_client: `batchserviceclient.BatchServiceClient`
:param block_blob_client: The storage block blob client to use.
:type block_blob_client: `azure.storage.blob.BlockBlobService`
:param str pool_id: The id of the pool to create.
:param str vm_size: vm size (sku)
:param int vm_count: number of vms to allocate
:param list app_files: The list of all the other scripts to upload.
"""
# pick the latest supported 16.04 sku for UbuntuServer
sku_to_use, image_ref_to_use = \
common.helpers.select_latest_verified_vm_image_with_node_agent_sku(
batch_client, 'Canonical', 'UbuntuServer', '14.04')
user = batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)
task_commands = get_list_from_file('configs/start_commands')
print(task_commands)
pool = batchmodels.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=image_ref_to_use, node_agent_sku_id=sku_to_use),
vm_size=vm_size,
target_dedicated=vm_count,
start_task=batchmodels.StartTask(
command_line=common.helpers.wrap_commands_in_shell(
'linux', task_commands),
user_identity=batchmodels.UserIdentity(auto_user=user),
resource_files=app_files,
wait_for_success=True))
common.helpers.create_pool_if_not_exist(batch_client, pool)
def add_task(batch_service_client, image_name, image_version, job_id, command,
name):
user = batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
elevation_level=batchmodels.ElevationLevel.admin,
scope=batchmodels.AutoUserScope.task,
))
task_id = name
task_container_settings = batch.models.TaskContainerSettings(
image_name=image_name + ":" + image_version,
container_run_options="--rm -p 4040:4040",
)
task = batch.models.TaskAddParameter(
id=task_id,
command_line=command,
container_settings=task_container_settings,
user_identity=user,
)
print("running " + command)
batch_service_client.task.add(job_id, task)
return task
def create_pool(batch_service_client, pool_id, users):
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str pool_id: An ID for the new pool.
:param str publisher: Marketplace image publisher
:param str offer: Marketplace image offer
:param str sku: Marketplace image sky
"""
print('Creating pool [{}]...'.format(pool_id))
# Create a new pool of Linux compute nodes using an Azure Virtual Machines
# Marketplace image. For more information about creating pools of Linux
# nodes, see:
# https://azure.microsoft.com/documentation/articles/batch-linux-nodes/
# The start task installs ffmpeg on each node from an available repository, using
# an administrator user identity.
new_pool = batch.models.PoolAddParameter(
id=pool_id,
user_accounts=users,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=batchmodels.ImageReference(publisher="Canonical",
offer="UbuntuServer",
sku="18.04-LTS",
version="latest"),
node_agent_sku_id="batch.node.ubuntu 18.04"),
vm_size=_POOL_VM_SIZE,
target_dedicated_nodes=_DEDICATED_POOL_NODE_COUNT,
target_low_priority_nodes=_LOW_PRIORITY_POOL_NODE_COUNT,
start_task=batchmodels.StartTask(
#command_line="/bin/bash -c \"git clone https://github.com/uiuc-arc/probfuzz.git; \
# cd probfuzz/; ./install_java.sh; ./install.sh\"",
# command_line="/bin/bash -c \"apt-get update\"",
command_line="/bin/bash -c \"sudo apt-get -y update; \
sudo apt-get install git; \
sudo apt-get install -y python2.7; \
sudo apt-get install -y python-pip; \
sudo apt-get install -y bc; \
sudo apt-get install -y r-base; \
sudo pip2 --no-cache-dir install pandas; \
sudo pip2 install rpy2==2.8.6;\
sudo pip2 install argparse;\
sudo pip2 install numpy;\
sudo pip2 install scipy;\
\"",
wait_for_success=True,
user_identity=batchmodels.UserIdentity(user_name="admin"),
#user_identity=batchmodels.UserIdentity(
# auto_user=batchmodels.AutoUserSpecification(
# scope=batchmodels.AutoUserScope.pool,
# elevation_level=batchmodels.ElevationLevel.admin)),
),
max_tasks_per_node=_MAX_TASKS_PER_NODE)
batch_service_client.pool.add(new_pool)
def add_task(self,
task_command: str,
task_name: str,
start_dir: str = None):
"""Add tasks to Azure Batch Job.
Parameters
----------
task_command : str
Task to run on job. This can be any task to run on the current job_id.
task_name : str
Name of task.
"""
user = batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
elevation_level=batchmodels.ElevationLevel.admin,
scope=batchmodels.AutoUserScope.task,
))
if (not start_dir and
self.config["POOL"]["PUBLISHER"] != "MicrosoftWindowsServer"):
start_dir = "/src"
if self.config["POOL"]["PUBLISHER"] == "MicrosoftWindowsServer":
if not start_dir:
start_dir = "src"
extra_opts = f"-w C:\\{start_dir}\\"
else:
extra_opts = f"--workdir {start_dir}/"
if self.use_fileshare:
if self.config["POOL"]["PUBLISHER"] == "MicrosoftWindowsServer":
mount = f"S:\\:C:\\{start_dir}\\logs"
else:
mount = f"/azfileshare/:{start_dir}/logs"
extra_opts += f" --volume {mount}"
self.task_id = task_name
logger.debug("Submitting task {0} to pool {1} with command {2}".format(
task_name, self.pool_id, task_command))
logger.debug(f"Extra configuration operations: {extra_opts}")
task_container_settings = batch.models.TaskContainerSettings(
image_name=self.image_name + ":" + self.image_version,
container_run_options=extra_opts,
)
task = batch.models.TaskAddParameter(
id=self.task_id,
command_line=task_command,
container_settings=task_container_settings,
environment_settings=[
batchmodels.EnvironmentSetting(name="SIM_WORKSPACE",
value=self.workspace),
batchmodels.EnvironmentSetting(name="SIM_ACCESS_KEY",
value=self.access_key),
],
user_identity=user,
)
self.batch_client.task.add(self.job_id, task)
def create_pool(batch_service_client, pool_id):
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str pool_id: An ID for the new pool.
:param str publisher: Marketplace image publisher
:param str offer: Marketplace image offer
:param str sku: Marketplace image sky
"""
print('Creating pool [{}]...'.format(pool_id))
# Create a new pool of Linux compute nodes using an Azure Virtual Machines
# Marketplace image. For more information about creating pools of Linux
# nodes, see:
# https://azure.microsoft.com/documentation/articles/batch-linux-nodes/
# The start task installs ffmpeg on each node from an available repository, using
# an administrator user identity.
#
task_commands = [
"apt-get update",
"apt-get -y install python3-pip",
"apt -y install htop",
"apt -y install iftop",
"pip3 install azure-storage-blob",
"pip3 install pyspark",
"pip3 install pandas",
"apt -y install openjdk-8-jre-headless"
]
new_pool = batch.models.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=batchmodels.ImageReference(
publisher="Canonical",
offer="UbuntuServer",
sku="18.04-LTS",
version="latest"
),
node_agent_sku_id="batch.node.ubuntu 18.04"),
vm_size=POOL_VM_SIZE,
target_dedicated_nodes=DEDICATED_POOL_NODE_COUNT,
target_low_priority_nodes=LOW_PRIORITY_POOL_NODE_COUNT,
start_task=batchmodels.StartTask(
command_line=wrap_commands_in_shell('linux',task_commands),
wait_for_success=True,
user_identity=batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)),
)
)
batch_service_client.pool.add(new_pool)
def add_tasks(batch_service_client, job_id, input_files,
output_container_sas_url):
"""
Adds a task for each input file in the collection to the specified job.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str job_id: The ID of the job to which to add the tasks.
:param list input_files: A collection of input files. One task will be
created for each input file.
:param output_container_sas_token: A SAS token granting write access to
the specified Azure Blob storage container.
"""
#print('Adding {} tasks to job [{}]...'.format(len(input_files), job_id))
print('Adding {} tasks to job [{}]...'.format(
_LOW_PRIORITY_POOL_NODE_COUNT, job_id))
tasks = list()
#for idx, input_file in enumerate(input_files):
for idx in range(_LOW_PRIORITY_POOL_NODE_COUNT):
# output_file_path='version{}.txt'.format(idx)
# command = "/bin/bash -c \"apt-get install git; \
# git clone https://github.com/uiuc-arc/probfuzz.git; \
# sudo apt-get install -y default-jdk;\
# cd ./probfuzz; \
# ./install.sh; \
# command = "/bin/bash -c \"git clone https://github.com/uiuc-arc/probfuzz.git; \
# cd ./probfuzz;\
# (cd language/antlr/ && wget http://www.antlr.org/download/antlr-4.7.1-complete.jar && ./run.sh); \
# ./check.py; \
# ./probfuzz.py 5\""
command = "/bin/bash -c \"cp -r /mnt/batch/tasks/startup/wd/probfuzz ./; \
cd probfuzz; \
./probfuzz.py 5\""
#command = "/bin/bash -c \"touch probfuzz.txt\""
tasks.append(
batch.models.TaskAddParameter(
id='Task{}'.format(idx),
command_line=command,
user_identity=batchmodels.UserIdentity(user_name="admin"),
#resource_files=input_files,
output_files=[
batchmodels.OutputFile(
file_pattern="./probfuzz/output/*",
destination=batchmodels.OutputFileDestination(
container=batchmodels.
OutputFileBlobContainerDestination(
container_url=output_container_sas_url)),
upload_options=batchmodels.OutputFileUploadOptions(
upload_condition=batchmodels.
OutputFileUploadCondition.task_success))
]))
batch_service_client.task.add_collection(job_id, tasks)
def create_pool(batch_service_client, pool_id,
resource_files, publisher, offer, sku,
vm_size, node_count):
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str pool_id: An ID for the new pool.
:param list resource_files: A collection of resource files for the pool's
start task.
:param str publisher: Marketplace image publisher
:param str offer: Marketplace image offer
:param str sku: Marketplace image sku
"""
print('Creating pool [{}]...'.format(pool_id))
# Create a new pool of Linux compute nodes using an Azure Virtual Machines
# Marketplace image. For more information about creating pools of Linux
# nodes, see:
# https://azure.microsoft.com/documentation/articles/batch-linux-nodes/
# Specify the commands for the pool's start task. The start task is run
# on each node as it joins the pool, and when it's rebooted or re-imaged.
# If there were installs needed, this is where we'd add them as
task_commands = ['echo starting up']
# Get the node agent SKU and image reference for the virtual machine
# configuration.
# For more information about the virtual machine configuration, see:
# https://azure.microsoft.com/documentation/articles/batch-linux-nodes/
sku_to_use, image_ref_to_use = \
select_latest_verified_vm_image_with_node_agent_sku(
batch_service_client, publisher, offer, sku)
user = batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)
new_pool = batch.models.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=image_ref_to_use,
node_agent_sku_id=sku_to_use),
vm_size=vm_size,
target_dedicated_nodes=node_count,
start_task=batch.models.StartTask(
command_line=wrap_commands_in_shell('linux',task_commands),
user_identity=batchmodels.UserIdentity(auto_user=user),
wait_for_success=True,
resource_files=resource_files),
)
try:
batch_service_client.pool.add(new_pool)
except batchmodels.batch_error.BatchErrorException as err:
print(err)
raise
def create_pool(self, pool_id):
"""
Creates a pool of compute nodes with the specified OS settings.
:param str pool_id: An ID for the new pool.
:param dict config: Configuration details.
"""
if pool_id in self.active_pools or self.pool_exists(pool_id):
return
self.logger.info("creating pool {}".format(pool_id))
pool_config = self.config['pools'][pool_id]
sku_to_use, image_ref_to_use = self.__get_vm_image_and_node_agent_sku(
pool_config)
start_vm_commands = None
if pool_config.get('create_vm_commands', None):
start_vm_commands = self.__create_commands(
pool_config['create_vm_commands'])
user = batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)
vm_configuration = batchmodels.VirtualMachineConfiguration(
image_reference=image_ref_to_use,
node_agent_sku_id=sku_to_use,
)
vm_start_task = batchmodels.StartTask(
command_line=self.__wrap_commands_in_shell('linux',
start_vm_commands),
user_identity=batchmodels.UserIdentity(auto_user=user),
wait_for_success=True)
new_pool = batchmodels.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=vm_configuration,
vm_size=pool_config['pool_vm_size'],
enable_auto_scale=True,
auto_scale_formula=pool_config['auto_scale_formula'],
auto_scale_evaluation_interval=datetime.timedelta(minutes=5),
start_task=vm_start_task,
max_tasks_per_node=pool_config['max_tasks_per_node'],
)
try:
self.batch_client.pool.add(new_pool)
except batchmodels.BatchErrorException as err:
self.__print_batch_exception(err)
raise
self.active_pools.add(pool_id)
def add_tasks(batch_service_client, job_id, input_files,
output_container_sas_url):
"""
Adds a task for each input file in the collection to the specified job.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str job_id: The ID of the job to which to add the tasks.
:param list input_files: A collection of input files. One task will be
created for each input file.
:param output_container_sas_token: A SAS token granting write access to
the specified Azure Blob storage container.
"""
#print('Adding {} tasks to job [{}]...'.format(len(input_files), job_id))
print('Adding {} tasks to job [{}]...'.format(
_LOW_PRIORITY_POOL_NODE_COUNT + _DEDICATED_POOL_NODE_COUNT, job_id))
tasks = list()
for idx, input_file in enumerate(input_files):
if idx >= 8:
break
input_file_path = input_file.file_path
input_file_name = input_file_path.split('/')[-1].split('.')[0]
#output_file_path="".join((input_file_path).split('.')[:-2]) + 'metrics_out_0319' + '.txt'
# TODO: cp task to stan dir; unzip; build; change mnt script; upload metrics & summary to storage
command = "/bin/bash -c \" \
time ./tar_to_metrics.sh {0} &> {1}_log.txt; \
\"".format(input_file_path, input_file_name)
#
#./run_metrics.sh ./metrics.py /mnt/batch/tasks/workitems/RunByIterJobAll/job-1/Task*/wd/example-models/; \
#./runner_10.sh /mnt/batch/tasks/workitems/RunByIterJobAll/job-1/Task*/wd/example-models/ _*.csv; \
#pip2 install --upgrade pip2; \
#sudo apt-get install -y r-base-core; \
#command = "/bin/bash -c \"touch probfuzz.txt\""
tasks.append(
batch.models.TaskAddParameter(
id='Task_{}'.format(input_file_name),
command_line=command,
user_identity=batchmodels.UserIdentity(user_name="admin"),
resource_files=input_files,
output_files=[
batchmodels.OutputFile(
file_pattern="*.txt".format(idx),
destination=batchmodels.OutputFileDestination(
container=batchmodels.
OutputFileBlobContainerDestination(
container_url=output_container_sas_url)),
upload_options=batchmodels.OutputFileUploadOptions(
upload_condition=batchmodels.
OutputFileUploadCondition.task_success))
]))
batch_service_client.task.add_collection(job_id, tasks)
def create_job(batch_client, name_job, name_pool, cmd_prep_task=None):
user = models.UserIdentity(auto_user=models.AutoUserSpecification(
elevation_level=models.ElevationLevel.admin,
scope=models.AutoUserScope.task))
prepare_task = models.JobPreparationTask(command_line=cmd_prep_task,
id=None,
user_identity=user)
job = models.JobAddParameter(
id=name_job,
pool_info=models.PoolInformation(pool_id=name_pool),
job_preparation_task=prepare_task)
batch_client.job.add(job)
def create_processing_pool(batch_service_client, start_task):
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:param str start_task: task start command.
:type batch_service_client: `azure.batch.BatchServiceClient`
"""
LOGGER.info(f'Creating pool [{PROCESSING_POOL_ID}]...')
image_ref_to_use = get_image_reference()
container_registry = \
batch_models.ContainerRegistry(
registry_server=REGISTRY_SERVER,
user_name=REGISTRY_ACCOUNT_USER,
password=REGISTRY_ACCOUNT_PASSWORD)
container_conf = batch_models.ContainerConfiguration(
container_image_names=[DOCKER_CONTAINER_URL],
container_registries=[container_registry])
new_pool = batch_models.PoolAddParameter(
id=PROCESSING_POOL_ID,
virtual_machine_configuration=
batch_models.VirtualMachineConfiguration(
image_reference=image_ref_to_use,
container_configuration=container_conf,
node_agent_sku_id=VM_AGENT_SKU),
vm_size=PROCESSING_POOL_VM_SIZE,
start_task=batch_models.StartTask(
command_line=start_task,
user_identity=batch_models.UserIdentity(
auto_user=batch_models.AutoUserSpecification(
scope='pool',
elevation_level='admin'))
),
enable_auto_scale=True,
auto_scale_evaluation_interval=datetime.timedelta(
minutes=PROCESSING_POOL_SCALE_INTERVAL_MINUTES),
auto_scale_formula=PROCESSING_POOL_SCALE_FORMULA)
try:
batch_service_client.pool.add(new_pool)
LOGGER.info("Processing Pool Created")
except batch_models.BatchErrorException as err:
if 'The specified pool already exists.' in err.error.message.value:
LOGGER.info("Pool already exists...")
else:
raise
def create_pool_and_wait_for_node(batch_client, pool_id, vm_size, vm_count,
sha1_cert_tp):
"""Creates an Azure Batch pool with the specified id.
:param batch_client: The batch client to use.
:type batch_client: `batchserviceclient.BatchServiceClient`
:param str pool_id: The id of the pool to create.
:param str vm_size: vm size (sku)
:param int vm_count: number of vms to allocate
:param str sha1_cert_tp: sha1 cert thumbprint for cert ref
"""
# pick the latest supported 16.04 sku for UbuntuServer
sku_to_use, image_ref_to_use = \
common.helpers.select_latest_verified_vm_image_with_node_agent_sku(
batch_client, 'Canonical', 'UbuntuServer', '16.04')
# create start task commands
# 1. update repository
# 2. install blobxfer pre-requisites
# 3. pip install blobxfer python script
start_task_commands = [
'apt-get update',
'apt-get install -y build-essential libssl-dev libffi-dev ' +
'libpython-dev python-dev python-pip', 'pip install --upgrade blobxfer'
]
user = batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)
# create pool with start task and cert ref with visibility of task
pool = batchmodels.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=image_ref_to_use, node_agent_sku_id=sku_to_use),
vm_size=vm_size,
target_dedicated=vm_count,
start_task=batchmodels.StartTask(
command_line=common.helpers.wrap_commands_in_shell(
'linux', start_task_commands),
user_identity=batchmodels.UserIdentity(auto_user=user),
wait_for_success=True),
certificate_references=[
batchmodels.CertificateReference(
sha1_cert_tp,
'sha1',
visibility=[batchmodels.CertificateVisibility.task])
],
)
common.helpers.create_pool_if_not_exist(batch_client, pool)
def add_task(
batch_service_client, job_id, task_id, num_instances,
application_cmdline, input_files, elevation_level,
output_file_names, output_container_sas,
coordination_cmdline, common_files):
"""
Adds a task for each input file in the collection to the specified job.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str job_id: The ID of the job to which to add the task.
:param str task_id: The ID of the task to be added.
:param str application_cmdline: The application commandline for the task.
:param list input_files: A collection of input files.
:param elevation_level: Elevation level used to run the task; either
'admin' or 'nonadmin'.
:type elevation_level: `azure.batch.models.ElevationLevel`
:param int num_instances: Number of instances for the task
:param str coordination_cmdline: The application commandline for the task.
:param list common_files: A collection of common input files.
"""
print('Adding {} task to job [{}]...'.format(task_id, job_id))
multi_instance_settings = None
if coordination_cmdline or (num_instances and num_instances > 1):
multi_instance_settings = batchmodels.MultiInstanceSettings(
number_of_instances=num_instances,
coordination_command_line=coordination_cmdline,
common_resource_files=common_files)
user = batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=elevation_level)
output_file = batchmodels.OutputFile(
file_pattern=output_file_names,
destination=batchmodels.OutputFileDestination(
container=batchmodels.OutputFileBlobContainerDestination(
container_url=output_container_sas)),
upload_options=batchmodels.OutputFileUploadOptions(
upload_condition=batchmodels.
OutputFileUploadCondition.task_completion))
task = batchmodels.TaskAddParameter(
id=task_id,
command_line=application_cmdline,
user_identity=batchmodels.UserIdentity(auto_user=user),
resource_files=input_files,
multi_instance_settings=multi_instance_settings,
output_files=[output_file])
batch_service_client.task.add(job_id, task)
def create_job(self, job_id, pool_id, total_nodes, is_linux_pool):
client = self._get_batch_client()
try:
pool_info = batchmodels.PoolInformation(pool_id=pool_id)
job = batchmodels.JobAddParameter(id=job_id, pool_info=pool_info)
try:
client.job.add(job)
except batchmodels.BatchErrorException as be:
if be.error and be.error.code == 'JobExists':
pass
else:
print('Error creating job, code={}, message={}'.format(
be.error.code, be.error.message))
raise
if is_linux_pool:
cmd_line = '/bin/bash -c azure-batch-ses.sh'
script = 'azure-batch-ses.sh'
script_url = 'https://raw.githubusercontent.com/Azure/azure-deadline/master/CloudProviderPlugin/Scripts/azure-batch-ses.sh'
else:
cmd_line = 'powershell.exe -file azure-batch-ses.ps1'
script = 'azure-batch-ses.ps1'
script_url = 'https://raw.githubusercontent.com/Azure/azure-deadline/master/CloudProviderPlugin/Scripts/azure-batch-ses.ps1'
task = batchmodels.TaskAddParameter(
id='',
command_line=cmd_line,
resource_files=[batchmodels.ResourceFile(script_url, script)],
constraints=batchmodels.TaskConstraints(
max_task_retry_count=3),
user_identity=batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)))
for i in range(total_nodes):
task.id = str(uuid.uuid4())
client.task.add(job_id=job.id, task=task)
except batchmodels.BatchErrorException as be:
if be.error:
print('Error creating job, code={}, message={}'.format(
be.error.code, be.error.message))
if be.error.values:
for e in be.error.values:
print('Key={}, Value={}'.format(e.key, e.value))
raise
def create_pool(batch_service_client, pool_id):
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str pool_id: An ID for the new pool.
:param str publisher: Marketplace image publisher
:param str offer: Marketplace image offer
:param str sku: Marketplace image sky
"""
print('Creating pool [{}]...'.format(pool_id))
# Create a new pool of Linux compute nodes using an Azure Virtual Machines
# Marketplace image. For more information about creating pools of Linux
# nodes, see:
# https://azure.microsoft.com/documentation/articles/batch-linux-nodes/
# The start task installs ffmpeg on each node from an available repository, using
# an administrator user identity.
new_pool = batch.models.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=batchmodels.ImageReference(publisher="Canonical",
offer="UbuntuServer",
sku="16.04-LTS",
version="latest"),
node_agent_sku_id="batch.node.ubuntu 16.04"),
vm_size=os.environ['POOL_VM_SIZE'],
target_dedicated_nodes=os.environ['DEDICATED_POOL_NODE_COUNT'],
target_low_priority_nodes=os.environ['LOW_PRIORITY_POOL_NODE_COUNT'],
start_task=batchmodels.StartTask(
command_line="/bin/bash -c \"apt-get update && \
apt-get install -y apt-transport-https ca-certificates curl software-properties-common &&\
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \
add-apt-repository 'deb [arch=amd64] https://download.docker.com/linux/ubuntu xenial stable' && \
apt-get update && \
apt-get install -y docker-ce\"",
wait_for_success=True,
user_identity=batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)),
))
batch_service_client.pool.add(new_pool)
def create_pool(batch_service_client, pool_id,
publisher, offer, sku, pool_vm_size, pool_node_count, low_priority_node_count, task_commands):
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str pool_id: An ID for the new pool.
:param list resource_files: A collection of resource files for the pool's
start task.
:param str publisher: Marketplace image publisher
:param str offer: Marketplace image offer
:param str sku: Marketplace image sku
"""
print('Creating pool [{}]...'.format(pool_id))
# Get the node agent SKU and image reference for the virtual machine
# configuration.
# For more information about the virtual machine configuration, see:
# https://azure.microsoft.com/documentation/articles/batch-linux-nodes/
sku_to_use, image_ref_to_use = \
common.helpers.select_latest_verified_vm_image_with_node_agent_sku(
batch_service_client, publisher, offer, sku)
user = batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)
new_pool = batch.models.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=image_ref_to_use,
node_agent_sku_id=sku_to_use),
vm_size=pool_vm_size,
target_dedicated_nodes=pool_node_count,
target_low_priority_nodes=low_priority_node_count,
start_task=batch.models.StartTask(
command_line=common.helpers.wrap_commands_in_shell('linux',
task_commands),
user_identity=batchmodels.UserIdentity(auto_user=user),
wait_for_success=True,
max_task_retry_count=-1)
)
try:
batch_service_client.pool.add(new_pool)
except batchmodels.batch_error.BatchErrorException as err:
print_batch_exception(err)
raise
def create_pool(batch_service_client, pool_id, application_files):
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str pool_id: An ID for the new pool.
:param str publisher: Marketplace image publisher
:param str offer: Marketplace image offer
:param str sku: Marketplace image sky
"""
print('Creating pool [{}]...'.format(pool_id))
# Create a new pool of Linux compute nodes using an Azure Virtual Machines
# Marketplace image. For more information about creating pools of Linux
# nodes, see:
# https://azure.microsoft.com/documentation/articles/batch-linux-nodes/
# The start task installs ffmpeg on each node from an available repository, using
# an administrator user identity.
new_pool = batch.models.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=batchmodels.ImageReference(
publisher="MicrosoftWindowsServer",
offer="WindowsServer",
sku="2016-Datacenter",
version="latest"
),
node_agent_sku_id="batch.node.windows amd64"),
vm_size=config._POOL_VM_SIZE,
target_dedicated_nodes=config._DEDICATED_POOL_NODE_COUNT,
target_low_priority_nodes=config._LOW_PRIORITY_POOL_NODE_COUNT,
start_task=batchmodels.StartTask(
command_line="cmd /c starttask.cmd",
resource_files=application_files,
wait_for_success=True,
user_identity=batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)),
)
)
batch_service_client.pool.add(new_pool)
def add_tasks(batch_service_client, job_id, input_files,
output_container_sas_url):
"""Adds a task for each pair with input file and command line in the collection to the specified job.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str job_id: The ID of the job to which to add the tasks.
:param list input_files: A collection of input files.
:param output_container_sas_url: A SAS token granting write access to
the specified Azure Blob storage container."""
print('Adding [{}] tasks to job [{}]'.format(_MAX_TASKS, job_id))
tasks = list()
mcr = "".join(input_files[-4].file_path)
program = "".join(input_files[-3].file_path)
data = "".join(input_files[-2].file_path)
design = "".join(input_files[-1].file_path)
output_file_pattern = '*.mat'
cogns = ['cogn_ps', 'cogn_po']
for idx in range(1, _MAX_TASKS + 1):
command = "/bin/bash -c 'sudo apt-get update -qq && sudo apt-get install libxt6 default-jre -y;tar xf {};" \
"mcr/install -destinationFolder /tmp/mcr18b -mode silent -agreeToLicense yes;rm -rf mcr {} /tmp/ma*" \
";./{} /tmp/mcr18b/v95 {} {} {} {} {} 15'".format(
mcr, mcr, program, data, design, cogns[1], 'batch' + str(idx) + '.mat', idx)
tasks.append(
batchmodels.TaskAddParameter(
id='Task{}'.format(idx),
command_line=command,
resource_files=input_files,
output_files=[
batchmodels.OutputFile(
file_pattern=output_file_pattern,
destination=batchmodels.OutputFileDestination(
container=batchmodels.
OutputFileBlobContainerDestination(
container_url=output_container_sas_url)),
upload_options=batchmodels.OutputFileUploadOptions(
upload_condition='taskCompletion'))
],
user_identity=batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin))))
batch_service_client.task.add_collection(job_id, tasks)
def add_tasks(batch_service_client, job_id, input_files,
output_container_sas_url):
"""Adds a task for each pair with input file and command line in the collection to the specified job.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str job_id: The ID of the job to which to add the tasks.
:param list input_files: A collection of input files. One task will be
created for each input file
:param output_container_sas_url: A SAS token granting write access to
the specified Azure Blob storage container."""
print('Adding [{}] tasks to job [{}]'.format(len(input_files) - 2, job_id))
tasks = list()
output_file_pattern = '*-[01w]*'
template = "".join(input_files[-1].file_path.split('.')[:-2])
script = "".join(input_files[-2].file_path)
for idx, input_file in enumerate(input_files[:-2]):
input_file_path = input_file.file_path
subjid = "".join(input_file_path.split('.')[:-2])
command = "/bin/bash -c 'source /usr/local/software/addpaths;./{} {} {}'".format(
script, template, subjid)
tasks.append(
batchmodels.TaskAddParameter(
id='Task{}'.format(idx + 1),
command_line=command,
resource_files=[input_file, input_files[-2], input_files[-1]],
output_files=[
batchmodels.OutputFile(
file_pattern=output_file_pattern,
destination=batchmodels.OutputFileDestination(
container=batchmodels.
OutputFileBlobContainerDestination(
container_url=output_container_sas_url)),
upload_options=batchmodels.OutputFileUploadOptions(
upload_condition='taskCompletion'))
],
user_identity=batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin))))
batch_service_client.task.add_collection(job_id, tasks)
def create_pool(batch_service_client, pool_id):
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str pool_id: An ID for the new pool.
:param str publisher: Marketplace image publisher
:param str offer: Marketplace image offer
:param str sku: Marketplace image sky
"""
print('Creating pool [{}]...'.format(pool_id))
# Create a new pool of Linux compute nodes using an Azure Virtual Machines
# Marketplace image. For more information about creating pools of Linux
# nodes, see:
# https://azure.microsoft.com/documentation/articles/batch-linux-nodes/
# The start task installs ffmpeg on each node from an available repository, using
# an administrator user identity.
new_pool = batch.models.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=batchmodels.ImageReference(publisher="Canonical",
offer="UbuntuServer",
sku="18.04-LTS",
version="latest"),
node_agent_sku_id="batch.node.ubuntu 18.04"),
vm_size=config._POOL_VM_SIZE,
target_dedicated_nodes=config._DEDICATED_POOL_NODE_COUNT,
target_low_priority_nodes=config._LOW_PRIORITY_POOL_NODE_COUNT,
start_task=batchmodels.StartTask(
command_line=
"/bin/bash -c \"apt-get update && apt-get install wget && wget http://cab.spbu.ru/files/release3.14.0/SPAdes-3.14.0-Linux.tar.gz && tar -xf SPAdes-3.14.0-Linux.tar.gz\"",
wait_for_success=True,
user_identity=batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)),
))
batch_service_client.pool.add(new_pool)
def create_pool(batch_service_client: batch.BatchServiceClient,
pool_id: str,
publisher: str = "Canonical",
offer: str = "UbuntuServer",
sku: str = "18.04-LTS") -> None:
"""
Creates a pool of compute nodes with the specified OS settings.
:param batch_service_client: A Batch service client.
:param pool_id: An ID for the new pool.
:param publisher: Marketplace image publisher
:param offer: Marketplace image offer
:param sku: Marketplace image sky
"""
print('Creating pool [{}]...'.format(pool_id))
# Create a new pool of Linux compute nodes using an Azure Virtual Machines
# Marketplace image. For more information about creating pools of Linux
# nodes, see:
# https://azure.microsoft.com/documentation/articles/batch-linux-nodes/
new_pool = batch.models.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=batchmodels.ImageReference(publisher=publisher,
offer=offer,
sku=sku,
version="latest"),
node_agent_sku_id="batch.node.ubuntu 18.04"),
vm_size=config._POOL_VM_SIZE,
target_dedicated_nodes=config._DEDICATED_POOL_NODE_COUNT,
target_low_priority_nodes=config._LOW_PRIORITY_POOL_NODE_COUNT,
start_task=batchmodels.StartTask(
command_line=
"/bin/bash -c \"apt-get update && apt-get -y install python3.7 python3-pip\"",
wait_for_success=True,
user_identity=batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)),
))
batch_service_client.pool.add(new_pool)
def add_tasks(batch_service_client, job_id, input_files,
output_container_sas_url, num_tasks):
"""Adds a task for each pair with input file and command line in the collection to the specified job.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str job_id: The ID of the job to which to add the tasks.
:param list input_files: A collection of input files.
:param output_container_sas_url: A SAS token granting write access to
the specified Azure Blob storage container.
:param num_tasks: Total number of tasks."""
print('Adding [{}] tasks to job [{}]'.format(num_tasks, job_id))
tasks = list()
output_file_pattern = '[ts]*'
for idx in range(num_tasks):
# input_file_path = input_file.file_path
# subjid = "".join(input_file_path.split('.')[:-2])
command = "/bin/bash -c 'source /usr/local/software/addpaths;" \
"randomise -i all_FA_skeletonised -m mean_FA_skeleton_mask -d design.mat -t design.con " \
"-D --T2 --uncorrp --seed={} -n {} -o {}_SEED{};cp -p ../stdout.txt stdout{}.txt'" \
"".format(idx+1, int(_RANDOMISE_ITER/_NUM_TASKS), _OUTPUT_PREFIX, idx+1, idx+1)
tasks.append(
batchmodels.TaskAddParameter(
id='Task{}'.format(idx + 1),
command_line=command,
resource_files=input_files,
output_files=[
batchmodels.OutputFile(
file_pattern=output_file_pattern,
destination=batchmodels.OutputFileDestination(
container=batchmodels.
OutputFileBlobContainerDestination(
container_url=output_container_sas_url)),
upload_options=batchmodels.OutputFileUploadOptions(
upload_condition='taskCompletion'))
],
user_identity=batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin))))
batch_service_client.task.add_collection(job_id, tasks)