def upload_file_to_container(blob_storage_service_client: BlobServiceClient, container_name: str, file_path: str) -> batchmodels.ResourceFile: """ Uploads a local file to an Azure Blob storage container. :param blob_storage_service_client: A blob service client. :param str container_name: The name of the Azure Blob storage container. :param str file_path: The local path to the file. :return: A ResourceFile initialized with a SAS URL appropriate for Batch tasks. """ blob_name = os.path.basename(file_path) blob_client = blob_storage_service_client.get_blob_client( container_name, blob_name) print(f'Uploading file {file_path} to container [{container_name}]...') with open(file_path, "rb") as data: blob_client.upload_blob(data, overwrite=True) sas_token = generate_blob_sas(config.STORAGE_ACCOUNT_NAME, container_name, blob_name, account_key=config.STORAGE_ACCOUNT_KEY, permission=BlobSasPermissions(read=True), expiry=datetime.datetime.utcnow() + datetime.timedelta(hours=2)) sas_url = generate_sas_url(config.STORAGE_ACCOUNT_NAME, config.STORAGE_ACCOUNT_DOMAIN, container_name, blob_name, sas_token) return batchmodels.ResourceFile(http_url=sas_url, file_path=blob_name)
def upload_file_to_container(block_blob_client, container_name, file_path): """ Uploads a local file to an Azure Blob storage container. :param block_blob_client: A blob service client. :type block_blob_client: `azure.storage.blob.BlockBlobService` :param str container_name: The name of the Azure Blob storage container. :param str file_path: The local path to the file. :rtype: `azure.batch.models.ResourceFile` :return: A ResourceFile initialized with a SAS URL appropriate for Batch tasks. """ blob_name = os.path.basename(file_path) print('Uploading file {} to container [{}]...'.format( file_path, container_name)) block_blob_client.create_blob_from_path(container_name, blob_name, file_path) # Obtain the SAS token for the container. sas_token = get_container_sas_token(block_blob_client, container_name, azureblob.BlobPermissions.READ) sas_url = block_blob_client.make_blob_url(container_name, blob_name, sas_token=sas_token) return batchmodels.ResourceFile(file_path=blob_name, blob_source=sas_url)
def upload_file_to_container(block_blob_client, container_name, file_path, timeout): """ Uploads a local file to an Azure Blob storage container. :param block_blob_client: A blob service client. :type block_blob_client: `azure.storage.blob.BlockBlobService` :param str container_name: The name of the Azure Blob storage container. :param str file_path: The local path to the file. :param int timeout: timeout in minutes from now for expiry, will only be used if expiry is not specified :rtype: `azure.batch.models.ResourceFile` :return: A ResourceFile initialized with a SAS URL appropriate for Batch tasks. """ blob_name = os.path.basename(file_path) print('Uploading file {} to container [{}]...'.format( file_path, container_name)) sas_url = upload_blob_and_create_sas(block_blob_client, container_name, blob_name, file_path, expiry=None, timeout=timeout) return batchmodels.ResourceFile(file_path=blob_name, http_url=sas_url)
def create_blob_resource( self, file_path, blob_name, job_file_path): """ Uploads a local file to an Azure Blob storage container for use as an azure batch resource. :param file_path: The local path to the file. :type file_path: str :param blob_name: path to upload to :type blob_name: str :param job_file_path: filepath for batch resource :type job_file_path: str :rtype: `azure.batch.models.ResourceFile` :return: A ResourceFile initialized with a SAS URL appropriate for Batch """ self.blob_client.upload_from_file( file_path, container_name=self.container_name, blob_name=blob_name ) sas_url = self.blob_client.generate_blob_url( container_name=self.container_name, blob_name=blob_name, add_sas=True ) return batchmodels.ResourceFile( file_path=job_file_path, http_url=sas_url )
def create_blob_batch_resource(block_blob_client, container_name, file_path, blob_name, job_file_path): """ Uploads a local file to an Azure Blob storage container for use as an azure batch resource. :param block_blob_client: A blob service client. :type block_blob_client: `azure.storage.blob.BlockBlobService` :param str container_name: The name of the Azure Blob storage container. :param str file_path: The local path to the file. :rtype: `azure.batch.models.ResourceFile` :return: A ResourceFile initialized with a SAS URL appropriate for Batch tasks. """ block_blob_client.create_blob_from_path(container_name, blob_name, file_path) sas_token = block_blob_client.generate_blob_shared_access_signature( container_name, blob_name, permission=azureblob.BlobPermissions.READ, expiry=datetime.datetime.utcnow() + datetime.timedelta(hours=120)) sas_url = block_blob_client.make_blob_url(container_name, blob_name, sas_token=sas_token) return batchmodels.ResourceFile(file_path=job_file_path, blob_source=sas_url)
def submit_job_and_add_task(batch_client, block_blob_client, job_id, pool_id): """Submits a job to the Azure Batch service and adds a task that runs a python script. :param batch_client: The batch client to use. :type batch_client: `batchserviceclient.BatchServiceClient` :param block_blob_client: The storage block blob client to use. :type block_blob_client: `azure.storage.blob.BlockBlobService` :param str job_id: The id of the job to create. :param str pool_id: The id of the pool to use. """ job = batchmodels.JobAddParameter( id=job_id, pool_info=batchmodels.PoolInformation(pool_id=pool_id)) batch_client.job.add(job) block_blob_client.create_container(_CONTAINER_NAME, fail_on_exist=False) sas_url = common.helpers.upload_blob_and_create_sas( block_blob_client, _CONTAINER_NAME, _SIMPLE_TASK_NAME, _SIMPLE_TASK_PATH, datetime.datetime.utcnow() + datetime.timedelta(hours=1)) task = batchmodels.TaskAddParameter( id="MyPythonTask", command_line="python " + _SIMPLE_TASK_NAME, resource_files=[ batchmodels.ResourceFile(file_path=_SIMPLE_TASK_NAME, http_url=sas_url) ]) batch_client.task.add(job_id=job.id, task=task)
def upload_file_to_container(container_client, file_path): """ Uploads a local file to an Azure Blob storage container. :param container_client: An Azure Blob storage container client. :type container_client: `azure.storage.blob.ContainerClient` :param str file_path: The local path to the file. :rtype: `azure.batch.models.ResourceFile` :return: A ResourceFile initialized with a SAS URL appropriate for Batch tasks. """ blob_name = os.path.basename(file_path) container_props = container_client.get_container_properties() container_name = container_props["name"] print('Uploading file {} to container [{}]...'.format( file_path, container_name)) blob_client = container_client.get_blob_client(blob_name) with open(file_path, "rb") as data: blob_client.upload_blob(data, blob_type="BlockBlob") sas_token = azureblob.generate_blob_sas( config._STORAGE_ACCOUNT_NAME, container_name, blob_name, account_key=config._STORAGE_ACCOUNT_KEY, permission=azureblob.BlobSasPermissions(read=True), expiry=datetime.datetime.utcnow() + datetime.timedelta(hours=2)) sas_url = 'https://{}.blob.core.windows.net/{}/{}?{}'.format( config._STORAGE_ACCOUNT_NAME, container_name, blob_name, sas_token) return batchmodels.ResourceFile(http_url=sas_url, file_path=blob_name)
def upload_file_to_container(block_blob_client, container_name, file_path): """ Uploads a local file to an Azure Blob storage container. :param block_blob_client: A blob service client. :type block_blob_client: `azure.storage.blob.BlockBlobService` :param str container_name: The name of the Azure Blob storage container. :param str file_path: The local path to the file. :rtype: `azure.batch.models.ResourceFile` :return: A ResourceFile initialized with a SAS URL appropriate for Batch tasks. """ blob_name = os.path.basename(file_path) print('Uploading file {} to container [{}]...'.format( file_path, container_name)) block_blob_client.create_blob_from_path(container_name, blob_name, file_path) sas_token = block_blob_client.generate_blob_shared_access_signature( container_name, blob_name, permission=azureblob.BlobPermissions.READ, expiry=datetime.datetime.utcnow() + datetime.timedelta(hours=2)) sas_url = block_blob_client.make_blob_url(container_name, blob_name, sas_token=sas_token) return batchmodels.ResourceFile(file_path=blob_name, blob_source=sas_url)
def get_resource_file(block_blob_client, container_name, blob_name, file_path): sas_url = common.helpers.upload_blob_and_create_sas( block_blob_client, container_name, blob_name, file_path, datetime.datetime.utcnow() + datetime.timedelta(hours=1)) logging.info('Uploading file {} from {} to container [{}]...'.format( blob_name, file_path, container_name)) return batchmodels.ResourceFile(file_path=blob_name, blob_source=sas_url)
def upload_file_to_container(block_blob_client: azureblob.BlockBlobService, container_name: str, file_path: str, folder: str = None) -> batchmodels.ResourceFile: """ Uploads a local file to an Azure Blob storage container. :param block_blob_client: A blob service client. :param container_name: The name of the Azure Blob storage container. :param file_path: The local path to the file. :param folder: The folder on container to store the file, default None. :return: A ResourceFile initialized with a SAS URL appropriate for Batch tasks. """ if folder: blob_name = f"{folder}/{os.path.basename(file_path)}" else: blob_name = os.path.basename(file_path) print('Uploading file {} to container [{}]...'.format( file_path, container_name)) block_blob_client.create_blob_from_path(container_name, blob_name, file_path) # Obtain the SAS token for the container. sas_token = get_container_sas_token(block_blob_client, container_name, azureblob.BlobPermissions.READ) sas_url = block_blob_client.make_blob_url(container_name, blob_name, sas_token=sas_token) return batchmodels.ResourceFile(file_path=blob_name, http_url=sas_url)
def upload_object_to_container(self, block_blob_client, container_name, blob_name, obj): """ Uploads a local file to an Azure Blob storage container. :param block_blob_client: A blob service client. :type block_blob_client: `azure.storage.blob.BlockBlobService` :param str container_name: The name of the Azure Blob storage container. :param str file_path: The local path to the file. :rtype: `azure.batch.models.ResourceFile` :return: A ResourceFile initialized with a SAS URL appropriate for Batch tasks. """ # print("Uploading file {} to container [{}]...".format(blob_name, container_name)) block_blob_client.create_blob_from_text(container_name, blob_name, dump(obj, Dumper=Dumper)) sas_token = block_blob_client.generate_blob_shared_access_signature( container_name, blob_name, permission=azureblob.BlobPermissions.READ, expiry=datetime.datetime.utcnow() + datetime.timedelta(hours=self.config.STORAGE_ACCESS_DURATION_HRS), ) sas_url = block_blob_client.make_blob_url(container_name, blob_name, sas_token=sas_token) return models.ResourceFile(http_url=sas_url, file_path=blob_name)
def get_resource_files(x_tiles, y_tiles, frame): """ Generate a list of resource files for the merge task. These will be the output tiles for the frame. :param x_tiles: Number of tiles on the X axis. :type x_tiles: int :param y_tiles: Number of tiles on the Y axis. :type y_tiles: int :param frame: Current frame :type frame: int """ tile_count = x_tiles * y_tiles output_sas = os.environ["OUTPUT_CONTAINER_SAS"] job_id = os.environ["AZ_BATCH_JOB_ID"] sas_parts = output_sas.split("?") files = [] for tile_name in get_tile_names(tile_count): path_to_file = "{}/outputs/frame-{}/{}".format( job_id, pad_number(frame, PAD_LEN_FRAME), tile_name) files.append( models.ResourceFile( "{}/{}?{}".format(sas_parts[0], path_to_file, sas_parts[1]), tile_name)) return files
def create_pool_and_wait_for_nodes(batch_client, block_blob_client, pool_id, vm_size, vm_count): """Creates an Azure Batch pool with the specified id. :param batch_client: The batch client to use. :type batch_client: `batchserviceclient.BatchServiceClient` :param block_blob_client: The storage block blob client to use. :type block_blob_client: `azure.storage.blob.BlockBlobService` :param str pool_id: The id of the pool to create. :param str vm_size: vm size (sku) :param int vm_count: number of vms to allocate :rtype: list :return: list of `batchserviceclient.models.ComputeNode` """ # pick the latest supported 14.04 sku for UbuntuServer sku_to_use, image_ref_to_use = \ common.helpers.select_latest_verified_vm_image_with_node_agent_sku( batch_client, 'Canonical', 'UbuntuServer', '14.04') # upload start task script block_blob_client.create_container(_CONTAINER_NAME, fail_on_exist=False) sas_url = common.helpers.upload_blob_and_create_sas( block_blob_client, _CONTAINER_NAME, _STARTTASK_RESOURCE_FILE, _STARTTASK_SHELL_SCRIPT_PATH, datetime.datetime.utcnow() + datetime.timedelta(hours=1)) # create pool with start task pool = batchmodels.CloudPool( id=pool_id, enable_inter_node_communication=True, virtual_machine_configuration=batchmodels.VirtualMachineConfiguration( image_reference=image_ref_to_use, node_agent_sku_id=sku_to_use), vm_size=vm_size, target_dedicated=vm_count, start_task=batchmodels.StartTask( command_line=_STARTTASK_RESOURCE_FILE, run_elevated=True, wait_for_success=True, resource_files=[ batchmodels.ResourceFile(file_path=_STARTTASK_RESOURCE_FILE, blob_source=sas_url) ]), ) common.helpers.create_pool_if_not_exist(batch_client, pool) # because we want all nodes to be available before any tasks are assigned # to the pool, here we will wait for all compute nodes to reach idle nodes = common.helpers.wait_for_all_nodes_state( batch_client, pool, frozenset((batchmodels.ComputeNodeState.starttaskfailed, batchmodels.ComputeNodeState.unusable, batchmodels.ComputeNodeState.idle))) # ensure all node are idle if any(node.state != batchmodels.ComputeNodeState.idle for node in nodes): raise RuntimeError('node(s) of pool {} not in idle state'.format( pool.id)) return nodes
def build_resource_file(storage_container, blob_prefix): """Generates a resource file object :param str storage_container: name of storage container attached to the batch account :param str blob_prefix: blob prefix to get resources files from. :rtype: batchmodels.ResourceFile """ return batchmodels.ResourceFile( auto_storage_container_name=storage_container, blob_prefix=blob_prefix)
def add_docker_batch_task(batch_client, block_blob_client, job_id, pool_id): """Submits a docker task via Batch scheduler :param batch_client: The batch client to use. :type batch_client: `batchserviceclient.BatchServiceClient` :param block_blob_client: The storage block blob client to use. :type block_blob_client: `azure.storage.blob.BlockBlobService` :param str job_id: The id of the job to use. :param str pool_id: The id of the pool to use. :rtype: list :return: a list of task_id of the task added. """ task_resource_sas_url = common.helpers.upload_blob_and_create_sas( block_blob_client, _CONTAINER_NAME, _TASK_RESOURCE_FILE, _TASK_RESOURCE_FILE_PATH, datetime.datetime.utcnow() + datetime.timedelta(hours=1)) output_container_sas_key = common.helpers.create_container_and_create_sas( block_blob_client=block_blob_client, container_name=_OUTPUT_CONTAINER_NAME, permission=azureblob.ContainerPermissions.WRITE | azureblob.ContainerPermissions.LIST, expiry=datetime.datetime.utcnow() + datetime.timedelta(hours=1)) # The start task pulls docker image yidingz/ffmpeg:v3 job = batchmodels.JobAddParameter( id=job_id, pool_info=batchmodels.PoolInformation(pool_id=pool_id), job_preparation_task=batchmodels.JobPreparationTask( command_line=_JOB_STARTTASK_CLI, run_elevated=True)) batch_client.job.add(job) task_id_list = [] index = 0 for url in _INPUT_FILE_URLS: filename = urllib.parse.urlsplit(url).path.split('/')[-1] parameters = "'{0}' '{1}' '{2}' '{3}'".format( url, filename, output_container_sas_key, block_blob_client.account_name) # Each task will download a video from chanel9, # transcode, and upload to specified output container task = batchmodels.TaskAddParameter( id=str(index).zfill(4) + '_' + filename.split('.')[0], command_line=_TASK_CLI.format(_TASK_RESOURCE_FILE, _FFMPEG_IMAGE, parameters), run_elevated=True, resource_files=[ batchmodels.ResourceFile(file_path=_TASK_RESOURCE_FILE, blob_source=task_resource_sas_url) ]) task_id_list.append(task.id) batch_client.task.add(job_id=job_id, task=task) index += 1 return task_id_list
def get_file_from_container(block_blob_client, container_name, file_path): blob_name = os.path.basename(file_path) # Obtain the SAS token for the container. sas_token = get_container_sas_token(block_blob_client, container_name, azureblob.BlobPermissions.READ) sas_url = block_blob_client.make_blob_url(container_name, blob_name, sas_token=sas_token) return batchmodels.ResourceFile(file_path=blob_name, http_url=sas_url)
def upload_file_to_container(block_blob_client, container_name, file_path, timeout): blob_name = os.path.basename(file_path) print('Uploading file {} to container [{}]...'.format( file_path, container_name)) sas_url = upload_blob_and_create_sas(block_blob_client, container_name, blob_name, file_path, expiry=None, timeout=timeout) return batchmodels.ResourceFile(file_path=blob_name, blob_source=sas_url)
def to_resource_file(self, dest: str = None) -> batch_models.ResourceFile: sas_token = self.blob_client.generate_blob_shared_access_signature( self.container, self.blob, permission=BlobPermissions.READ, expiry=datetime.datetime.utcnow() + datetime.timedelta(days=365)) sas_url = self.blob_client.make_blob_url(self.container, self.blob, sas_token=sas_token) return batch_models.ResourceFile(file_path=dest or self.dest, blob_source=sas_url)
def create_post_processing_task(container_settings, job_id, tasks): post_processing_task = batchmodels.TaskAddParameter( id="postprocessing", command_line=f'/bin/sh -c "cat {job_id}/output/*.csv"', # oddly, using storage_container_url doesn't work here # but storage container name does resource_files=[batchmodels.ResourceFile(auto_storage_container_name=_CONTAINER_NAME, blob_prefix=f"{job_id}/output/", file_path="")], container_settings=container_settings, depends_on=batchmodels.TaskDependencies(task_ids=[task.id for task in tasks]) ) return post_processing_task
def create_processing_tasks(block_blob_client, job_id, container_settings): tasks = [] resource_files = [] container_url = azure_helpers.create_container_sas( block_blob_client, _CONTAINER_NAME, datetime.datetime.utcnow() + datetime.timedelta(hours=24) ) for task_script in ["petal_width.py", "sepal_width.py"]: task_name = task_script.replace("_", "") task_path = Path("resources", task_script) script_stem = task_script.rstrip(".py") # Shared access signature, allows access to a resource using a token for a specified amount of time sas_url = azure_helpers.upload_blob_and_create_sas( block_blob_client, _CONTAINER_NAME, task_name, task_path, datetime.datetime.utcnow() + datetime.timedelta(hours=1)) resource_file = batchmodels.ResourceFile( file_path=task_name, http_url=sas_url) resource_files.append(resource_file) # output data to upload to storage container output_file = batchmodels.OutputFile( file_pattern=f'output/{script_stem}.csv', destination=batchmodels.OutputFileDestination( container=batchmodels.OutputFileBlobContainerDestination( container_url=container_url, path=f'{job_id}/output/{script_stem}.csv' ) ), upload_options=batchmodels.OutputFileUploadOptions( upload_condition=batchmodels.OutputFileUploadCondition.task_completion ) ) # add tasks with resources and outputs tasks.append(batchmodels.TaskAddParameter( id=script_stem, command_line=f'python3 {task_name}', resource_files=[resource_file], container_settings=container_settings, output_files=[output_file] )) return resource_files, tasks
def create_job(self, job_id, pool_id, total_nodes, is_linux_pool): client = self._get_batch_client() try: pool_info = batchmodels.PoolInformation(pool_id=pool_id) job = batchmodels.JobAddParameter(id=job_id, pool_info=pool_info) try: client.job.add(job) except batchmodels.BatchErrorException as be: if be.error and be.error.code == 'JobExists': pass else: print('Error creating job, code={}, message={}'.format( be.error.code, be.error.message)) raise if is_linux_pool: cmd_line = '/bin/bash -c azure-batch-ses.sh' script = 'azure-batch-ses.sh' script_url = 'https://raw.githubusercontent.com/Azure/azure-deadline/master/CloudProviderPlugin/Scripts/azure-batch-ses.sh' else: cmd_line = 'powershell.exe -file azure-batch-ses.ps1' script = 'azure-batch-ses.ps1' script_url = 'https://raw.githubusercontent.com/Azure/azure-deadline/master/CloudProviderPlugin/Scripts/azure-batch-ses.ps1' task = batchmodels.TaskAddParameter( id='', command_line=cmd_line, resource_files=[batchmodels.ResourceFile(script_url, script)], constraints=batchmodels.TaskConstraints( max_task_retry_count=3), user_identity=batchmodels.UserIdentity( auto_user=batchmodels.AutoUserSpecification( scope=batchmodels.AutoUserScope.pool, elevation_level=batchmodels.ElevationLevel.admin))) for i in range(total_nodes): task.id = str(uuid.uuid4()) client.task.add(job_id=job.id, task=task) except batchmodels.BatchErrorException as be: if be.error: print('Error creating job, code={}, message={}'.format( be.error.code, be.error.message)) if be.error.values: for e in be.error.values: print('Key={}, Value={}'.format(e.key, e.value)) raise
def create_pool_and_wait_for_nodes(batch_client, block_blob_client, pool_id, vm_size, vm_count): sku_to_use, image_ref_to_use = common.helpers.select_latest_verified_vm_image_with_node_agent_sku( batch_client, 'Canonical', 'UbuntuServer', '14.04') block_blob_client.create_container(_CONTAINER_NAME, fail_on_exist=False) # upload start task script block_blob_client.create_container(_CONTAINER_NAME, fail_on_exist=False) sas_url = common.helpers.upload_blob_and_create_sas( block_blob_client, _CONTAINER_NAME, _STARTTASK_RESOURCE_FILE, _STARTTASK_SHELL_SCRIPT_PATH, datetime.datetime.utcnow() + datetime.timedelta(hours=1)) # create pool and execute starttask pool = batchmodels.PoolAddParameter( id=pool_id, enable_inter_node_communication=True, virtual_machine_configuration=batchmodels.VirtualMachineConfiguration( image_reference=image_ref_to_use, node_agent_sku_id=sku_to_use), vm_size=vm_size, target_dedicated=vm_count, start_task=batchmodels.StartTask( command_line=_STARTTASK_RESOURCE_FILE, run_elevated=True, wait_for_success=True, resource_files=[ batchmodels.ResourceFile(file_path=_STARTTASK_RESOURCE_FILE, blob_source=sas_url) ]), ) common.helpers.create_pool_if_not_exist(batch_client, pool) # because we want all nodes to be available before any tasks are assigned # to the pool, here we will wait for all compute nodes to reach idle nodes = common.helpers.wait_for_all_nodes_state( batch_client, pool, frozenset((batchmodels.ComputeNodeState.starttaskfailed, batchmodels.ComputeNodeState.unusable, batchmodels.ComputeNodeState.idle))) # ensure all node are idle if any(node.state != batchmodels.ComputeNodeState.idle for node in nodes): raise RuntimeError('node(s) of pool {} not in idle state'.format( pool.id)) return nodes
def upload_text_to_container(container_name: str, application_name: str, content: str, file_path: str, blob_client=None) -> batch_models.ResourceFile: blob_name = file_path blob_path = application_name + "/" + blob_name # + '/' + time_stamp + '/' + blob_name blob_client.create_container(container_name, fail_on_exist=False) blob_client.create_blob_from_text(container_name, blob_path, content) sas_token = blob_client.generate_blob_shared_access_signature( container_name, blob_path, permission=blob.BlobPermissions.READ, expiry=datetime.datetime.utcnow() + datetime.timedelta(days=365), ) sas_url = blob_client.make_blob_url(container_name, blob_path, sas_token=sas_token) return batch_models.ResourceFile(file_path=blob_name, blob_source=sas_url)
def upload_file_to_container(container_name, application_name, file_path, blob_client=None, use_full_path=False, node_path=None) -> batch_models.ResourceFile: """ Uploads a local file to an Azure Blob storage container. :param blob_client: A blob service client. :type blocblob_clientk_blob_client: `azure.storage.blob.BlockBlobService` :param str container_name: The name of the Azure Blob storage container. :param str file_path: The local path to the file. :param str node_path: Path on the local node. By default will be the same as file_path :rtype: `azure.batch.models.ResourceFile` :return: A ResourceFile initialized with a SAS URL appropriate for Batch tasks. """ file_path = normalize_path(file_path) blob_name = None if use_full_path: blob_name = file_path.strip("/") else: blob_name = os.path.basename(file_path) blob_path = application_name + "/" + blob_name if not node_path: node_path = blob_name blob_client.create_container(container_name, fail_on_exist=False) blob_client.create_blob_from_path(container_name, blob_path, file_path) sas_token = blob_client.generate_blob_shared_access_signature( container_name, blob_path, permission=blob.BlobPermissions.READ, expiry=datetime.datetime.utcnow() + datetime.timedelta(days=7), ) sas_url = blob_client.make_blob_url(container_name, blob_path, sas_token=sas_token) return batch_models.ResourceFile(file_path=node_path, blob_source=sas_url)
def get_resource_file_list_from_container(block_blob_client, container): """Creates a Resource File for each blob in the storage container :param block_blob_client: The storage block blob client to use. :type block_blob_client: `azure.storage.blob.BlockBlobService` :param str container: The name of the storage container """ blob_list = block_blob_client.list_blobs(container) result = [] for blob in blob_list: print("appending " + blob.name) sas_url = common.helpers.create_sas_url( block_blob_client, container, blob.name, azureblob.BlobPermissions.READ, datetime.datetime.utcnow() + datetime.timedelta(hours=1)) result.append( batchmodels.ResourceFile(file_path=blob.name, http_url=sas_url)) return result
def create_pool(batch_client, block_blob_client, pool_id, vm_size, vm_count): """Creates an Azure Batch pool with the specified id. :param batch_client: The batch client to use. :type batch_client: `batchserviceclient.BatchServiceClient` :param block_blob_client: The storage block blob client to use. :type block_blob_client: `azure.storage.blob.BlockBlobService` :param str pool_id: The id of the pool to create. :param str vm_size: vm size (sku) :param int vm_count: number of vms to allocate """ # pick the latest supported 14.04 sku for UbuntuServer sku_to_use, image_ref_to_use = \ common.helpers.select_latest_verified_vm_image_with_node_agent_sku( batch_client, 'Canonical', 'UbuntuServer', '14.04') block_blob_client.create_container( _CONTAINER_NAME, fail_on_exist=False) sas_url = common.helpers.upload_blob_and_create_sas( block_blob_client, _CONTAINER_NAME, _SIMPLE_TASK_NAME, _SIMPLE_TASK_PATH, datetime.datetime.utcnow() + datetime.timedelta(hours=1)) pool = batchmodels.PoolAddParameter( id=pool_id, virtual_machine_configuration=batchmodels.VirtualMachineConfiguration( image_reference=image_ref_to_use, node_agent_sku_id=sku_to_use), vm_size=vm_size, target_dedicated=vm_count, start_task=batchmodels.StartTask( command_line="python " + _SIMPLE_TASK_NAME, resource_files=[batchmodels.ResourceFile( file_path=_SIMPLE_TASK_NAME, blob_source=sas_url)])) common.helpers.create_pool_if_not_exist(batch_client, pool)
def uploadFile2Blob(container_client, container_name, upload_file_path, loaded): # local file name filename = os.path.basename(upload_file_path) print("\nUploading to Azure Storage as blob:\n\t" + filename) if not loaded: # Upload the created file with open(upload_file_path, "rb") as data: container_client.upload_blob(filename, data=data) # blob_url = blob_client.getBlobUrl() account_name = os.environ.get('AZURE_BLOB_ACCOUNT_NAME') # blob_url = f"https://{account_name}.blob.core.windows.net/{container_name}/{blob_name}" container_url = blob_url = f"https://{account_name}.blob.core.windows.net/{container_name}" # print(blob_url, filename) print(container_url, filename) # return batchmodels.ResourceFile(http_url=blob_url, file_path=filename) return batchmodels.ResourceFile(storage_container_url=container_url)
def create_resource_file_from_blob(block_blob_client, container_name, blob_name): """ Creates a resource file object from a given blob client, container, and blob :param block_blob_client: A blob client with credentials. :type blob_blob_client: 'azure.storage.blob.BlockBlobService' :param str conatiner_name: The container name. :param str blob_name: The blob name. """ sas_token = block_blob_client.generate_blob_shared_access_signature( container_name = container_name, blob_name = blob_name, permission=azureblob.BlobPermissions.READ, expiry=datetime.datetime.utcnow() + datetime.timedelta(hours=2) ) sas_url = block_blob_client.make_blob_url(container_name, blob_name, sas_token=sas_token) return batchmodels.ResourceFile(file_path=blob_name,blob_source=sas_url)
def submit_job_and_add_task(batch_client, block_blob_client, job_id, pool_id): job = batchmodels.JobAddParameter( id=job_id, pool_info=batchmodels.PoolInformation(pool_id=pool_id)) batch_client.job.add(job) block_blob_client.create_container(CONTAINER_NAME, fail_on_exist=False) sas_url = upload_blob_and_create_sas( block_blob_client, CONTAINER_NAME, TASK_NAME, TASK_PATH, datetime.datetime.utcnow() + datetime.timedelta(hours=1)) task = batchmodels.TaskAddParameter(id="SliceTask", command_line="python3 " + TASK_NAME, resource_files=[ batchmodels.ResourceFile( file_path=TASK_NAME, blob_source=sas_url) ]) batch_client.task.add(job_id=job.id, task=task)
def task_submit(task_name): ''' Automatic task submission to Azure. Pool, VMs, Jobs should be created in advance. ''' common_resource_files = [] for folder, _, files in os.walk('../'): # Skip setup folder if os.path.abspath(folder) == os.path.abspath('./'): continue for file_name in files: if file_name.endswith('.py') or file_name.endswith('.ini'): blob_url = os.path.join(config_azure['storage_account_url'], file_name) file_path = os.path.join(os.path.basename(folder), file_name) print('Mapping {} to {}'.format(blob_url, file_path)) common_resource_files.append( batchmodel.ResourceFile(blob_url, file_path, file_mode='0775')) command = '/usr/lib64/openmpi/bin/mpirun -mca btl_tcp_if_include eth0 -oversubscribe -n {0} -host $AZ_BATCH_HOST_LIST -wd $AZ_BATCH_TASK_SHARED_DIR python36 $AZ_BATCH_TASK_SHARED_DIR/bench.py'.format( config_azure['task_number_of_procs']) coordination_command = '/bin/bash -c "echo $AZ_BATCH_HOST_LIST; echo $AZ_BATCH_TASK_SHARED_DIR; echo $AZ_BATCH_MASTER_NODE;"' multi_instance_settings = batchmodel.MultiInstanceSettings( coordination_command_line=coordination_command, number_of_instances=config_azure['task_number_of_instances'], common_resource_files=common_resource_files) user = batchmodel.UserIdentity(auto_user=batchmodel.AutoUserSpecification( scope=batchmodel.AutoUserScope.pool, elevation_level=batchmodel.ElevationLevel.non_admin)) task = batchmodel.TaskAddParameter( id=task_name, command_line=command, multi_instance_settings=multi_instance_settings, user_identity=user) print('Adding bench tasks to job [{0}]...'.format(config_azure['job_id'])) batch_service.task.add(config_azure['job_id'], task)