def __init__(self, batch_storage_account):
# Create a Batch service client. We'll now be interacting with the Batch
# service in addition to Storage
self.my_storage = batch_storage_account
configuration = AzureCredentials()
self.account_name = configuration.getBatchAccountName()
self.account_key = configuration.getBatchAccountKey()
self.account_url = configuration.getBatchAccountUrl()
self.credentials = batchauth.SharedKeyCredentials(self.account_name,self.account_key)
self.batch_client = batch.BatchServiceClient(self.credentials,batch_url=self.account_url)
print("API version is: ", self.batch_client.task.api_version)
#self.batch_client.task.api_version = "2020-03-01.11.0"
#print("API version is: ", self.batch_client.task.api_version)
batch_config = AzureBatchConfiguration()
self.pool_count = batch_config.getNodeCount()
self.pool_type = batch_config.getVMSize()
self.pool_os = batch_config.getOSType()
self.pool_publisher = batch_config.getOSPublisher()
self.pool_os_ver = batch_config.getOSVersion()
self.pool_engine_name = batch_config.getEngineName()
batch_json = find_file_path("batch.json", "../")
print("Found batch.json in: {}".format(batch_json))
credential_json = find_file_path("credentials.json", "../")
print("Found credentials.json in: {}".format(credential_json))
task_json = find_file_path("task.json", "../")
print("Found task.json in: {}".format(task_json))
self.my_storage.addApplicationFilePath("engine/"+batch_config.getEngineName())
self.my_storage.addApplicationFilePath("engine/taskengine.py")
#self.my_storage.addApplicationFilePath("batchwrapper/batch.json")
self.my_storage.addApplicationFilePath(batch_json)
self.my_storage.addApplicationFilePath("batchwrapper/__init__.py")
#self.my_storage.addApplicationFilePath("batchwrapper/credentials.json")
self.my_storage.addApplicationFilePath(credential_json)
self.my_storage.addApplicationFilePath(task_json)
self.my_storage.addApplicationFilePath("batchwrapper/config.py")
self.my_storage.uploadApplicationFiles()
def addFileToUpload(self, file_name=''):
#/mnt/batch/tasks/workitems/<job id>/job-<#>/<task id>/wd
#/mnt/batch/tasks/shared
name = find_file_path(file_name, "../")
print("Found file to upload: {}".format(name))
if name != '':
self.file_list_to_upload.extend([name])
print("Will upload: {}".format(self.file_list_to_upload))
def create_pool(self, app_resources='', app_name='', input_resources='', task_files=''):
random = getRandomizer()
self.pool_name = 'azpool_' + random
print('Creating pool [{}]...'.format(self.pool_name))
if app_resources == '':
print("App resources cannot be empty. HINT: you get this object from AzureBatchStorage.getAppResourceFiles")
exit(-1)
if app_name == '':
print("App name cannot be empty. HINT: This python file needs to inherit from AzureBatchEngine")
exit(-1)
task_commands = [
'mkdir -p $AZ_BATCH_NODE_SHARED_DIR/batchwrapper',
'mkdir -p $AZ_BATCH_NODE_SHARED_DIR/engine',
'mkdir -p $AZ_BATCH_NODE_SHARED_DIR/tasks',
'chmod 777 $AZ_BATCH_NODE_SHARED_DIR/tasks',
'cp -p {} $AZ_BATCH_NODE_SHARED_DIR/engine/'.format(self.pool_engine_name),
'cp -p {} $AZ_BATCH_NODE_SHARED_DIR/engine/'.format("taskengine.py"),
'cp -p {} $AZ_BATCH_NODE_SHARED_DIR/batchwrapper/'.format("credentials.json"),
'cp -p {} $AZ_BATCH_NODE_SHARED_DIR/batchwrapper/'.format("batch.json"),
'cp -p {} $AZ_BATCH_NODE_SHARED_DIR/tasks'.format("task.json"),
'cp -p {} $AZ_BATCH_NODE_SHARED_DIR/batchwrapper/'.format("config.py"),
'cp -p {} $AZ_BATCH_NODE_SHARED_DIR/batchwrapper/'.format("__init__.py"),
'cp -p {} $AZ_BATCH_NODE_SHARED_DIR/engine/'.format("__init__.py"),
'cp -p {} $AZ_BATCH_NODE_SHARED_DIR/tasks/'.format("__init__.py"),
'cp -p {} $AZ_BATCH_NODE_SHARED_DIR/'.format("__init__.py"),
]
for j in task_files:
print("adding application: {}".format(j.file_path))
task_commands.extend(['cp -p {} $AZ_BATCH_NODE_SHARED_DIR/tasks'.format(j.file_path)])
for i in input_resources:
print("adding file: {}".format(i.file_path))
task_commands.extend(['cp -p {} $AZ_BATCH_NODE_SHARED_DIR'.format(i.file_path)])
requirements_file = find_file_path("requirements.txt", ".")
print("Found requirements.txt in: {}".format(requirements_file))
if(requirements_file != None):
task_commands.extend( ['/bin/bash -c "sudo yum -y install java-11-openjdk python3"', 'sudo pip3 install -r '+ requirements_file])
###task_commands.extend( ['curl -fSsL https://bootstrap.pypa.io/3.4/get-pip.py | python3', 'pip3 install -r '+ requirements_file])
else:
task_commands.extend( ['/bin/bash -c "sudo yum -y install java-11-openjdk python3"', 'sudo pip3 install -Iv azure-storage-blob==12.3.0 azure-batch==9.0.0 azure-servicebus==0.50.0'])
print("commands to be published: {}".format(task_commands))
# Get the node agent SKU and image reference for the virtual machine
# configuration.
# For more information about the virtual machine configuration, see:
# https://azure.microsoft.com/documentation/articles/batch-linux-nodes/
sku_to_use, image_ref_to_use = \
common.helpers.select_latest_verified_vm_image_with_node_agent_sku(self.batch_client, self.pool_publisher, self.pool_os, self.pool_os_ver)
user = batchmodels.AutoUserSpecification(scope=batchmodels.AutoUserScope.pool, elevation_level=batchmodels.ElevationLevel.admin)
resource_meta = list()
resource_meta.extend(app_resources)
resource_meta.extend(task_files)
resource_meta.extend(input_resources)
new_pool = batch.models.PoolAddParameter(id=self.pool_name,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=image_ref_to_use,
node_agent_sku_id=sku_to_use),
vm_size=self.pool_type,
target_dedicated_nodes=self.pool_count,
max_tasks_per_node=1,
task_scheduling_policy=batch.models.TaskSchedulingPolicy(node_fill_type=batch.models.ComputeNodeFillType.spread),
start_task=batch.models.StartTask(
command_line=common.helpers.wrap_commands_in_shell('linux',
task_commands),
user_identity=batchmodels.UserIdentity(auto_user=user),
wait_for_success=True,
resource_files=resource_meta),
)
try:
self.batch_client.pool.add(new_pool)
except batchmodels.batch_error.BatchErrorException as err:
print(err)
raise
print("Waiting for pool to become ready after creation")
self._wait_for_ready_pool(self.pool_name)
time.sleep(self.pool_count * 1.5)
print("Back up - ready to work now")
return self.pool_name