def upload_file_to_container(blob_storage_service_client: BlobServiceClient,
container_name: str,
file_path: str) -> batchmodels.ResourceFile:
"""
Uploads a local file to an Azure Blob storage container.
:param blob_storage_service_client: A blob service client.
:param str container_name: The name of the Azure Blob storage container.
:param str file_path: The local path to the file.
:return: A ResourceFile initialized with a SAS URL appropriate for Batch
tasks.
"""
blob_name = os.path.basename(file_path)
blob_client = blob_storage_service_client.get_blob_client(
container_name, blob_name)
print(f'Uploading file {file_path} to container [{container_name}]...')
with open(file_path, "rb") as data:
blob_client.upload_blob(data, overwrite=True)
sas_token = generate_blob_sas(config.STORAGE_ACCOUNT_NAME,
container_name,
blob_name,
account_key=config.STORAGE_ACCOUNT_KEY,
permission=BlobSasPermissions(read=True),
expiry=datetime.datetime.utcnow() +
datetime.timedelta(hours=2))
sas_url = generate_sas_url(config.STORAGE_ACCOUNT_NAME,
config.STORAGE_ACCOUNT_DOMAIN, container_name,
blob_name, sas_token)
return batchmodels.ResourceFile(http_url=sas_url, file_path=blob_name)
def upload_file_to_container(block_blob_client, container_name, file_path):
"""
Uploads a local file to an Azure Blob storage container.
:param block_blob_client: A blob service client.
:type block_blob_client: `azure.storage.blob.BlockBlobService`
:param str container_name: The name of the Azure Blob storage container.
:param str file_path: The local path to the file.
:rtype: `azure.batch.models.ResourceFile`
:return: A ResourceFile initialized with a SAS URL appropriate for Batch
tasks.
"""
blob_name = os.path.basename(file_path)
print('Uploading file {} to container [{}]...'.format(
file_path, container_name))
block_blob_client.create_blob_from_path(container_name, blob_name,
file_path)
# Obtain the SAS token for the container.
sas_token = get_container_sas_token(block_blob_client, container_name,
azureblob.BlobPermissions.READ)
sas_url = block_blob_client.make_blob_url(container_name,
blob_name,
sas_token=sas_token)
return batchmodels.ResourceFile(file_path=blob_name, blob_source=sas_url)
def upload_file_to_container(block_blob_client, container_name, file_path,
timeout):
"""
Uploads a local file to an Azure Blob storage container.
:param block_blob_client: A blob service client.
:type block_blob_client: `azure.storage.blob.BlockBlobService`
:param str container_name: The name of the Azure Blob storage container.
:param str file_path: The local path to the file.
:param int timeout: timeout in minutes from now for expiry,
will only be used if expiry is not specified
:rtype: `azure.batch.models.ResourceFile`
:return: A ResourceFile initialized with a SAS URL appropriate for Batch
tasks.
"""
blob_name = os.path.basename(file_path)
print('Uploading file {} to container [{}]...'.format(
file_path, container_name))
sas_url = upload_blob_and_create_sas(block_blob_client,
container_name,
blob_name,
file_path,
expiry=None,
timeout=timeout)
return batchmodels.ResourceFile(file_path=blob_name, http_url=sas_url)
def create_blob_resource(
self, file_path, blob_name, job_file_path):
"""
Uploads a local file to an Azure Blob storage container for use as
an azure batch resource.
:param file_path: The local path to the file.
:type file_path: str
:param blob_name: path to upload to
:type blob_name: str
:param job_file_path: filepath for batch resource
:type job_file_path: str
:rtype: `azure.batch.models.ResourceFile`
:return: A ResourceFile initialized with a SAS URL appropriate for Batch
"""
self.blob_client.upload_from_file(
file_path, container_name=self.container_name, blob_name=blob_name
)
sas_url = self.blob_client.generate_blob_url(
container_name=self.container_name, blob_name=blob_name, add_sas=True
)
return batchmodels.ResourceFile(
file_path=job_file_path,
http_url=sas_url
)
def create_blob_batch_resource(block_blob_client, container_name, file_path,
blob_name, job_file_path):
"""
Uploads a local file to an Azure Blob storage container for use as
an azure batch resource.
:param block_blob_client: A blob service client.
:type block_blob_client: `azure.storage.blob.BlockBlobService`
:param str container_name: The name of the Azure Blob storage container.
:param str file_path: The local path to the file.
:rtype: `azure.batch.models.ResourceFile`
:return: A ResourceFile initialized with a SAS URL appropriate for Batch
tasks.
"""
block_blob_client.create_blob_from_path(container_name, blob_name,
file_path)
sas_token = block_blob_client.generate_blob_shared_access_signature(
container_name,
blob_name,
permission=azureblob.BlobPermissions.READ,
expiry=datetime.datetime.utcnow() + datetime.timedelta(hours=120))
sas_url = block_blob_client.make_blob_url(container_name,
blob_name,
sas_token=sas_token)
return batchmodels.ResourceFile(file_path=job_file_path,
blob_source=sas_url)
def submit_job_and_add_task(batch_client, block_blob_client, job_id, pool_id):
"""Submits a job to the Azure Batch service and adds
a task that runs a python script.
:param batch_client: The batch client to use.
:type batch_client: `batchserviceclient.BatchServiceClient`
:param block_blob_client: The storage block blob client to use.
:type block_blob_client: `azure.storage.blob.BlockBlobService`
:param str job_id: The id of the job to create.
:param str pool_id: The id of the pool to use.
"""
job = batchmodels.JobAddParameter(
id=job_id, pool_info=batchmodels.PoolInformation(pool_id=pool_id))
batch_client.job.add(job)
block_blob_client.create_container(_CONTAINER_NAME, fail_on_exist=False)
sas_url = common.helpers.upload_blob_and_create_sas(
block_blob_client, _CONTAINER_NAME, _SIMPLE_TASK_NAME,
_SIMPLE_TASK_PATH,
datetime.datetime.utcnow() + datetime.timedelta(hours=1))
task = batchmodels.TaskAddParameter(
id="MyPythonTask",
command_line="python " + _SIMPLE_TASK_NAME,
resource_files=[
batchmodels.ResourceFile(file_path=_SIMPLE_TASK_NAME,
http_url=sas_url)
])
batch_client.task.add(job_id=job.id, task=task)
def upload_file_to_container(container_client, file_path):
"""
Uploads a local file to an Azure Blob storage container.
:param container_client: An Azure Blob storage container client.
:type container_client: `azure.storage.blob.ContainerClient`
:param str file_path: The local path to the file.
:rtype: `azure.batch.models.ResourceFile`
:return: A ResourceFile initialized with a SAS URL appropriate for Batch
tasks.
"""
blob_name = os.path.basename(file_path)
container_props = container_client.get_container_properties()
container_name = container_props["name"]
print('Uploading file {} to container [{}]...'.format(
file_path, container_name))
blob_client = container_client.get_blob_client(blob_name)
with open(file_path, "rb") as data:
blob_client.upload_blob(data, blob_type="BlockBlob")
sas_token = azureblob.generate_blob_sas(
config._STORAGE_ACCOUNT_NAME,
container_name,
blob_name,
account_key=config._STORAGE_ACCOUNT_KEY,
permission=azureblob.BlobSasPermissions(read=True),
expiry=datetime.datetime.utcnow() + datetime.timedelta(hours=2))
sas_url = 'https://{}.blob.core.windows.net/{}/{}?{}'.format(
config._STORAGE_ACCOUNT_NAME, container_name, blob_name, sas_token)
return batchmodels.ResourceFile(http_url=sas_url, file_path=blob_name)
def upload_file_to_container(block_blob_client, container_name, file_path):
"""
Uploads a local file to an Azure Blob storage container.
:param block_blob_client: A blob service client.
:type block_blob_client: `azure.storage.blob.BlockBlobService`
:param str container_name: The name of the Azure Blob storage container.
:param str file_path: The local path to the file.
:rtype: `azure.batch.models.ResourceFile`
:return: A ResourceFile initialized with a SAS URL appropriate for Batch
tasks.
"""
blob_name = os.path.basename(file_path)
print('Uploading file {} to container [{}]...'.format(
file_path, container_name))
block_blob_client.create_blob_from_path(container_name, blob_name,
file_path)
sas_token = block_blob_client.generate_blob_shared_access_signature(
container_name,
blob_name,
permission=azureblob.BlobPermissions.READ,
expiry=datetime.datetime.utcnow() + datetime.timedelta(hours=2))
sas_url = block_blob_client.make_blob_url(container_name,
blob_name,
sas_token=sas_token)
return batchmodels.ResourceFile(file_path=blob_name, blob_source=sas_url)
def get_resource_file(block_blob_client, container_name, blob_name, file_path):
sas_url = common.helpers.upload_blob_and_create_sas(
block_blob_client, container_name, blob_name, file_path,
datetime.datetime.utcnow() + datetime.timedelta(hours=1))
logging.info('Uploading file {} from {} to container [{}]...'.format(
blob_name, file_path, container_name))
return batchmodels.ResourceFile(file_path=blob_name, blob_source=sas_url)
def upload_file_to_container(block_blob_client: azureblob.BlockBlobService,
container_name: str,
file_path: str,
folder: str = None) -> batchmodels.ResourceFile:
"""
Uploads a local file to an Azure Blob storage container.
:param block_blob_client: A blob service client.
:param container_name: The name of the Azure Blob storage container.
:param file_path: The local path to the file.
:param folder: The folder on container to store the file, default None.
:return: A ResourceFile initialized with a SAS URL appropriate for Batch
tasks.
"""
if folder:
blob_name = f"{folder}/{os.path.basename(file_path)}"
else:
blob_name = os.path.basename(file_path)
print('Uploading file {} to container [{}]...'.format(
file_path, container_name))
block_blob_client.create_blob_from_path(container_name, blob_name,
file_path)
# Obtain the SAS token for the container.
sas_token = get_container_sas_token(block_blob_client, container_name,
azureblob.BlobPermissions.READ)
sas_url = block_blob_client.make_blob_url(container_name,
blob_name,
sas_token=sas_token)
return batchmodels.ResourceFile(file_path=blob_name, http_url=sas_url)
def upload_object_to_container(self, block_blob_client, container_name,
blob_name, obj):
"""
Uploads a local file to an Azure Blob storage container.
:param block_blob_client: A blob service client.
:type block_blob_client: `azure.storage.blob.BlockBlobService`
:param str container_name: The name of the Azure Blob storage container.
:param str file_path: The local path to the file.
:rtype: `azure.batch.models.ResourceFile`
:return: A ResourceFile initialized with a SAS URL appropriate for Batch
tasks.
"""
# print("Uploading file {} to container [{}]...".format(blob_name, container_name))
block_blob_client.create_blob_from_text(container_name, blob_name,
dump(obj, Dumper=Dumper))
sas_token = block_blob_client.generate_blob_shared_access_signature(
container_name,
blob_name,
permission=azureblob.BlobPermissions.READ,
expiry=datetime.datetime.utcnow() +
datetime.timedelta(hours=self.config.STORAGE_ACCESS_DURATION_HRS),
)
sas_url = block_blob_client.make_blob_url(container_name,
blob_name,
sas_token=sas_token)
return models.ResourceFile(http_url=sas_url, file_path=blob_name)
def get_resource_files(x_tiles, y_tiles, frame):
"""
Generate a list of resource files for the merge task. These will be the
output tiles for the frame.
:param x_tiles: Number of tiles on the X axis.
:type x_tiles: int
:param y_tiles: Number of tiles on the Y axis.
:type y_tiles: int
:param frame: Current frame
:type frame: int
"""
tile_count = x_tiles * y_tiles
output_sas = os.environ["OUTPUT_CONTAINER_SAS"]
job_id = os.environ["AZ_BATCH_JOB_ID"]
sas_parts = output_sas.split("?")
files = []
for tile_name in get_tile_names(tile_count):
path_to_file = "{}/outputs/frame-{}/{}".format(
job_id, pad_number(frame, PAD_LEN_FRAME), tile_name)
files.append(
models.ResourceFile(
"{}/{}?{}".format(sas_parts[0], path_to_file, sas_parts[1]),
tile_name))
return files
def create_pool_and_wait_for_nodes(batch_client, block_blob_client, pool_id,
vm_size, vm_count):
"""Creates an Azure Batch pool with the specified id.
:param batch_client: The batch client to use.
:type batch_client: `batchserviceclient.BatchServiceClient`
:param block_blob_client: The storage block blob client to use.
:type block_blob_client: `azure.storage.blob.BlockBlobService`
:param str pool_id: The id of the pool to create.
:param str vm_size: vm size (sku)
:param int vm_count: number of vms to allocate
:rtype: list
:return: list of `batchserviceclient.models.ComputeNode`
"""
# pick the latest supported 14.04 sku for UbuntuServer
sku_to_use, image_ref_to_use = \
common.helpers.select_latest_verified_vm_image_with_node_agent_sku(
batch_client, 'Canonical', 'UbuntuServer', '14.04')
# upload start task script
block_blob_client.create_container(_CONTAINER_NAME, fail_on_exist=False)
sas_url = common.helpers.upload_blob_and_create_sas(
block_blob_client, _CONTAINER_NAME, _STARTTASK_RESOURCE_FILE,
_STARTTASK_SHELL_SCRIPT_PATH,
datetime.datetime.utcnow() + datetime.timedelta(hours=1))
# create pool with start task
pool = batchmodels.CloudPool(
id=pool_id,
enable_inter_node_communication=True,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=image_ref_to_use, node_agent_sku_id=sku_to_use),
vm_size=vm_size,
target_dedicated=vm_count,
start_task=batchmodels.StartTask(
command_line=_STARTTASK_RESOURCE_FILE,
run_elevated=True,
wait_for_success=True,
resource_files=[
batchmodels.ResourceFile(file_path=_STARTTASK_RESOURCE_FILE,
blob_source=sas_url)
]),
)
common.helpers.create_pool_if_not_exist(batch_client, pool)
# because we want all nodes to be available before any tasks are assigned
# to the pool, here we will wait for all compute nodes to reach idle
nodes = common.helpers.wait_for_all_nodes_state(
batch_client, pool,
frozenset((batchmodels.ComputeNodeState.starttaskfailed,
batchmodels.ComputeNodeState.unusable,
batchmodels.ComputeNodeState.idle)))
# ensure all node are idle
if any(node.state != batchmodels.ComputeNodeState.idle for node in nodes):
raise RuntimeError('node(s) of pool {} not in idle state'.format(
pool.id))
return nodes
def build_resource_file(storage_container, blob_prefix):
"""Generates a resource file object
:param str storage_container: name of storage container attached to the batch account
:param str blob_prefix: blob prefix to get resources files from.
:rtype: batchmodels.ResourceFile
"""
return batchmodels.ResourceFile(
auto_storage_container_name=storage_container,
blob_prefix=blob_prefix)
def add_docker_batch_task(batch_client, block_blob_client, job_id, pool_id):
"""Submits a docker task via Batch scheduler
:param batch_client: The batch client to use.
:type batch_client: `batchserviceclient.BatchServiceClient`
:param block_blob_client: The storage block blob client to use.
:type block_blob_client: `azure.storage.blob.BlockBlobService`
:param str job_id: The id of the job to use.
:param str pool_id: The id of the pool to use.
:rtype: list
:return: a list of task_id of the task added.
"""
task_resource_sas_url = common.helpers.upload_blob_and_create_sas(
block_blob_client, _CONTAINER_NAME, _TASK_RESOURCE_FILE,
_TASK_RESOURCE_FILE_PATH,
datetime.datetime.utcnow() + datetime.timedelta(hours=1))
output_container_sas_key = common.helpers.create_container_and_create_sas(
block_blob_client=block_blob_client,
container_name=_OUTPUT_CONTAINER_NAME,
permission=azureblob.ContainerPermissions.WRITE
| azureblob.ContainerPermissions.LIST,
expiry=datetime.datetime.utcnow() + datetime.timedelta(hours=1))
# The start task pulls docker image yidingz/ffmpeg:v3
job = batchmodels.JobAddParameter(
id=job_id,
pool_info=batchmodels.PoolInformation(pool_id=pool_id),
job_preparation_task=batchmodels.JobPreparationTask(
command_line=_JOB_STARTTASK_CLI, run_elevated=True))
batch_client.job.add(job)
task_id_list = []
index = 0
for url in _INPUT_FILE_URLS:
filename = urllib.parse.urlsplit(url).path.split('/')[-1]
parameters = "'{0}' '{1}' '{2}' '{3}'".format(
url, filename, output_container_sas_key,
block_blob_client.account_name)
# Each task will download a video from chanel9,
# transcode, and upload to specified output container
task = batchmodels.TaskAddParameter(
id=str(index).zfill(4) + '_' + filename.split('.')[0],
command_line=_TASK_CLI.format(_TASK_RESOURCE_FILE, _FFMPEG_IMAGE,
parameters),
run_elevated=True,
resource_files=[
batchmodels.ResourceFile(file_path=_TASK_RESOURCE_FILE,
blob_source=task_resource_sas_url)
])
task_id_list.append(task.id)
batch_client.task.add(job_id=job_id, task=task)
index += 1
return task_id_list
def get_file_from_container(block_blob_client, container_name, file_path):
blob_name = os.path.basename(file_path)
# Obtain the SAS token for the container.
sas_token = get_container_sas_token(block_blob_client, container_name,
azureblob.BlobPermissions.READ)
sas_url = block_blob_client.make_blob_url(container_name,
blob_name,
sas_token=sas_token)
return batchmodels.ResourceFile(file_path=blob_name, http_url=sas_url)
def upload_file_to_container(block_blob_client, container_name, file_path,
timeout):
blob_name = os.path.basename(file_path)
print('Uploading file {} to container [{}]...'.format(
file_path, container_name))
sas_url = upload_blob_and_create_sas(block_blob_client,
container_name,
blob_name,
file_path,
expiry=None,
timeout=timeout)
return batchmodels.ResourceFile(file_path=blob_name, blob_source=sas_url)
def to_resource_file(self, dest: str = None) -> batch_models.ResourceFile:
sas_token = self.blob_client.generate_blob_shared_access_signature(
self.container,
self.blob,
permission=BlobPermissions.READ,
expiry=datetime.datetime.utcnow() + datetime.timedelta(days=365))
sas_url = self.blob_client.make_blob_url(self.container,
self.blob,
sas_token=sas_token)
return batch_models.ResourceFile(file_path=dest or self.dest,
blob_source=sas_url)
def create_post_processing_task(container_settings, job_id, tasks):
post_processing_task = batchmodels.TaskAddParameter(
id="postprocessing",
command_line=f'/bin/sh -c "cat {job_id}/output/*.csv"',
# oddly, using storage_container_url doesn't work here
# but storage container name does
resource_files=[batchmodels.ResourceFile(auto_storage_container_name=_CONTAINER_NAME,
blob_prefix=f"{job_id}/output/",
file_path="")],
container_settings=container_settings,
depends_on=batchmodels.TaskDependencies(task_ids=[task.id for task in tasks])
)
return post_processing_task
def create_processing_tasks(block_blob_client, job_id, container_settings):
tasks = []
resource_files = []
container_url = azure_helpers.create_container_sas(
block_blob_client,
_CONTAINER_NAME,
datetime.datetime.utcnow() + datetime.timedelta(hours=24)
)
for task_script in ["petal_width.py", "sepal_width.py"]:
task_name = task_script.replace("_", "")
task_path = Path("resources", task_script)
script_stem = task_script.rstrip(".py")
# Shared access signature, allows access to a resource using a token for a specified amount of time
sas_url = azure_helpers.upload_blob_and_create_sas(
block_blob_client,
_CONTAINER_NAME,
task_name,
task_path,
datetime.datetime.utcnow() + datetime.timedelta(hours=1))
resource_file = batchmodels.ResourceFile(
file_path=task_name,
http_url=sas_url)
resource_files.append(resource_file)
# output data to upload to storage container
output_file = batchmodels.OutputFile(
file_pattern=f'output/{script_stem}.csv',
destination=batchmodels.OutputFileDestination(
container=batchmodels.OutputFileBlobContainerDestination(
container_url=container_url,
path=f'{job_id}/output/{script_stem}.csv'
)
),
upload_options=batchmodels.OutputFileUploadOptions(
upload_condition=batchmodels.OutputFileUploadCondition.task_completion
)
)
# add tasks with resources and outputs
tasks.append(batchmodels.TaskAddParameter(
id=script_stem,
command_line=f'python3 {task_name}',
resource_files=[resource_file],
container_settings=container_settings,
output_files=[output_file]
))
return resource_files, tasks
def create_job(self, job_id, pool_id, total_nodes, is_linux_pool):
client = self._get_batch_client()
try:
pool_info = batchmodels.PoolInformation(pool_id=pool_id)
job = batchmodels.JobAddParameter(id=job_id, pool_info=pool_info)
try:
client.job.add(job)
except batchmodels.BatchErrorException as be:
if be.error and be.error.code == 'JobExists':
pass
else:
print('Error creating job, code={}, message={}'.format(
be.error.code, be.error.message))
raise
if is_linux_pool:
cmd_line = '/bin/bash -c azure-batch-ses.sh'
script = 'azure-batch-ses.sh'
script_url = 'https://raw.githubusercontent.com/Azure/azure-deadline/master/CloudProviderPlugin/Scripts/azure-batch-ses.sh'
else:
cmd_line = 'powershell.exe -file azure-batch-ses.ps1'
script = 'azure-batch-ses.ps1'
script_url = 'https://raw.githubusercontent.com/Azure/azure-deadline/master/CloudProviderPlugin/Scripts/azure-batch-ses.ps1'
task = batchmodels.TaskAddParameter(
id='',
command_line=cmd_line,
resource_files=[batchmodels.ResourceFile(script_url, script)],
constraints=batchmodels.TaskConstraints(
max_task_retry_count=3),
user_identity=batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)))
for i in range(total_nodes):
task.id = str(uuid.uuid4())
client.task.add(job_id=job.id, task=task)
except batchmodels.BatchErrorException as be:
if be.error:
print('Error creating job, code={}, message={}'.format(
be.error.code, be.error.message))
if be.error.values:
for e in be.error.values:
print('Key={}, Value={}'.format(e.key, e.value))
raise
def create_pool_and_wait_for_nodes(batch_client, block_blob_client, pool_id,
vm_size, vm_count):
sku_to_use, image_ref_to_use = common.helpers.select_latest_verified_vm_image_with_node_agent_sku(
batch_client, 'Canonical', 'UbuntuServer', '14.04')
block_blob_client.create_container(_CONTAINER_NAME, fail_on_exist=False)
# upload start task script
block_blob_client.create_container(_CONTAINER_NAME, fail_on_exist=False)
sas_url = common.helpers.upload_blob_and_create_sas(
block_blob_client, _CONTAINER_NAME, _STARTTASK_RESOURCE_FILE,
_STARTTASK_SHELL_SCRIPT_PATH,
datetime.datetime.utcnow() + datetime.timedelta(hours=1))
# create pool and execute starttask
pool = batchmodels.PoolAddParameter(
id=pool_id,
enable_inter_node_communication=True,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=image_ref_to_use, node_agent_sku_id=sku_to_use),
vm_size=vm_size,
target_dedicated=vm_count,
start_task=batchmodels.StartTask(
command_line=_STARTTASK_RESOURCE_FILE,
run_elevated=True,
wait_for_success=True,
resource_files=[
batchmodels.ResourceFile(file_path=_STARTTASK_RESOURCE_FILE,
blob_source=sas_url)
]),
)
common.helpers.create_pool_if_not_exist(batch_client, pool)
# because we want all nodes to be available before any tasks are assigned
# to the pool, here we will wait for all compute nodes to reach idle
nodes = common.helpers.wait_for_all_nodes_state(
batch_client, pool,
frozenset((batchmodels.ComputeNodeState.starttaskfailed,
batchmodels.ComputeNodeState.unusable,
batchmodels.ComputeNodeState.idle)))
# ensure all node are idle
if any(node.state != batchmodels.ComputeNodeState.idle for node in nodes):
raise RuntimeError('node(s) of pool {} not in idle state'.format(
pool.id))
return nodes
def upload_text_to_container(container_name: str, application_name: str, content: str, file_path: str,
blob_client=None) -> batch_models.ResourceFile:
blob_name = file_path
blob_path = application_name + "/" + blob_name # + '/' + time_stamp + '/' + blob_name
blob_client.create_container(container_name, fail_on_exist=False)
blob_client.create_blob_from_text(container_name, blob_path, content)
sas_token = blob_client.generate_blob_shared_access_signature(
container_name,
blob_path,
permission=blob.BlobPermissions.READ,
expiry=datetime.datetime.utcnow() + datetime.timedelta(days=365),
)
sas_url = blob_client.make_blob_url(container_name, blob_path, sas_token=sas_token)
return batch_models.ResourceFile(file_path=blob_name, blob_source=sas_url)
def upload_file_to_container(container_name,
application_name,
file_path,
blob_client=None,
use_full_path=False,
node_path=None) -> batch_models.ResourceFile:
"""
Uploads a local file to an Azure Blob storage container.
:param blob_client: A blob service client.
:type blocblob_clientk_blob_client: `azure.storage.blob.BlockBlobService`
:param str container_name: The name of the Azure Blob storage container.
:param str file_path: The local path to the file.
:param str node_path: Path on the local node. By default will be the same as file_path
:rtype: `azure.batch.models.ResourceFile`
:return: A ResourceFile initialized with a SAS URL appropriate for Batch
tasks.
"""
file_path = normalize_path(file_path)
blob_name = None
if use_full_path:
blob_name = file_path.strip("/")
else:
blob_name = os.path.basename(file_path)
blob_path = application_name + "/" + blob_name
if not node_path:
node_path = blob_name
blob_client.create_container(container_name, fail_on_exist=False)
blob_client.create_blob_from_path(container_name, blob_path, file_path)
sas_token = blob_client.generate_blob_shared_access_signature(
container_name,
blob_path,
permission=blob.BlobPermissions.READ,
expiry=datetime.datetime.utcnow() + datetime.timedelta(days=7),
)
sas_url = blob_client.make_blob_url(container_name,
blob_path,
sas_token=sas_token)
return batch_models.ResourceFile(file_path=node_path, blob_source=sas_url)
def get_resource_file_list_from_container(block_blob_client, container):
"""Creates a Resource File for each blob in the storage container
:param block_blob_client: The storage block blob client to use.
:type block_blob_client: `azure.storage.blob.BlockBlobService`
:param str container: The name of the storage container
"""
blob_list = block_blob_client.list_blobs(container)
result = []
for blob in blob_list:
print("appending " + blob.name)
sas_url = common.helpers.create_sas_url(
block_blob_client, container, blob.name,
azureblob.BlobPermissions.READ,
datetime.datetime.utcnow() + datetime.timedelta(hours=1))
result.append(
batchmodels.ResourceFile(file_path=blob.name, http_url=sas_url))
return result
def create_pool(batch_client, block_blob_client, pool_id, vm_size, vm_count):
"""Creates an Azure Batch pool with the specified id.
:param batch_client: The batch client to use.
:type batch_client: `batchserviceclient.BatchServiceClient`
:param block_blob_client: The storage block blob client to use.
:type block_blob_client: `azure.storage.blob.BlockBlobService`
:param str pool_id: The id of the pool to create.
:param str vm_size: vm size (sku)
:param int vm_count: number of vms to allocate
"""
# pick the latest supported 14.04 sku for UbuntuServer
sku_to_use, image_ref_to_use = \
common.helpers.select_latest_verified_vm_image_with_node_agent_sku(
batch_client, 'Canonical', 'UbuntuServer', '14.04')
block_blob_client.create_container(
_CONTAINER_NAME,
fail_on_exist=False)
sas_url = common.helpers.upload_blob_and_create_sas(
block_blob_client,
_CONTAINER_NAME,
_SIMPLE_TASK_NAME,
_SIMPLE_TASK_PATH,
datetime.datetime.utcnow() + datetime.timedelta(hours=1))
pool = batchmodels.PoolAddParameter(
id=pool_id,
virtual_machine_configuration=batchmodels.VirtualMachineConfiguration(
image_reference=image_ref_to_use,
node_agent_sku_id=sku_to_use),
vm_size=vm_size,
target_dedicated=vm_count,
start_task=batchmodels.StartTask(
command_line="python " + _SIMPLE_TASK_NAME,
resource_files=[batchmodels.ResourceFile(
file_path=_SIMPLE_TASK_NAME,
blob_source=sas_url)]))
common.helpers.create_pool_if_not_exist(batch_client, pool)
def uploadFile2Blob(container_client, container_name, upload_file_path,
loaded):
# local file name
filename = os.path.basename(upload_file_path)
print("\nUploading to Azure Storage as blob:\n\t" + filename)
if not loaded:
# Upload the created file
with open(upload_file_path, "rb") as data:
container_client.upload_blob(filename, data=data)
# blob_url = blob_client.getBlobUrl()
account_name = os.environ.get('AZURE_BLOB_ACCOUNT_NAME')
# blob_url = f"https://{account_name}.blob.core.windows.net/{container_name}/{blob_name}"
container_url = blob_url = f"https://{account_name}.blob.core.windows.net/{container_name}"
# print(blob_url, filename)
print(container_url, filename)
# return batchmodels.ResourceFile(http_url=blob_url, file_path=filename)
return batchmodels.ResourceFile(storage_container_url=container_url)
def create_resource_file_from_blob(block_blob_client, container_name, blob_name):
"""
Creates a resource file object from a given blob client, container, and blob
:param block_blob_client: A blob client with credentials.
:type blob_blob_client: 'azure.storage.blob.BlockBlobService'
:param str conatiner_name: The container name.
:param str blob_name: The blob name.
"""
sas_token = block_blob_client.generate_blob_shared_access_signature(
container_name = container_name,
blob_name = blob_name,
permission=azureblob.BlobPermissions.READ,
expiry=datetime.datetime.utcnow() + datetime.timedelta(hours=2)
)
sas_url = block_blob_client.make_blob_url(container_name,
blob_name,
sas_token=sas_token)
return batchmodels.ResourceFile(file_path=blob_name,blob_source=sas_url)
def submit_job_and_add_task(batch_client, block_blob_client, job_id, pool_id):
job = batchmodels.JobAddParameter(
id=job_id, pool_info=batchmodels.PoolInformation(pool_id=pool_id))
batch_client.job.add(job)
block_blob_client.create_container(CONTAINER_NAME, fail_on_exist=False)
sas_url = upload_blob_and_create_sas(
block_blob_client, CONTAINER_NAME, TASK_NAME, TASK_PATH,
datetime.datetime.utcnow() + datetime.timedelta(hours=1))
task = batchmodels.TaskAddParameter(id="SliceTask",
command_line="python3 " + TASK_NAME,
resource_files=[
batchmodels.ResourceFile(
file_path=TASK_NAME,
blob_source=sas_url)
])
batch_client.task.add(job_id=job.id, task=task)
def task_submit(task_name):
'''
Automatic task submission to Azure. Pool, VMs, Jobs should be created in advance.
'''
common_resource_files = []
for folder, _, files in os.walk('../'):
# Skip setup folder
if os.path.abspath(folder) == os.path.abspath('./'):
continue
for file_name in files:
if file_name.endswith('.py') or file_name.endswith('.ini'):
blob_url = os.path.join(config_azure['storage_account_url'],
file_name)
file_path = os.path.join(os.path.basename(folder), file_name)
print('Mapping {} to {}'.format(blob_url, file_path))
common_resource_files.append(
batchmodel.ResourceFile(blob_url,
file_path,
file_mode='0775'))
command = '/usr/lib64/openmpi/bin/mpirun -mca btl_tcp_if_include eth0 -oversubscribe -n {0} -host $AZ_BATCH_HOST_LIST -wd $AZ_BATCH_TASK_SHARED_DIR python36 $AZ_BATCH_TASK_SHARED_DIR/bench.py'.format(
config_azure['task_number_of_procs'])
coordination_command = '/bin/bash -c "echo $AZ_BATCH_HOST_LIST; echo $AZ_BATCH_TASK_SHARED_DIR; echo $AZ_BATCH_MASTER_NODE;"'
multi_instance_settings = batchmodel.MultiInstanceSettings(
coordination_command_line=coordination_command,
number_of_instances=config_azure['task_number_of_instances'],
common_resource_files=common_resource_files)
user = batchmodel.UserIdentity(auto_user=batchmodel.AutoUserSpecification(
scope=batchmodel.AutoUserScope.pool,
elevation_level=batchmodel.ElevationLevel.non_admin))
task = batchmodel.TaskAddParameter(
id=task_name,
command_line=command,
multi_instance_settings=multi_instance_settings,
user_identity=user)
print('Adding bench tasks to job [{0}]...'.format(config_azure['job_id']))
batch_service.task.add(config_azure['job_id'], task)
