def main(req: func.HttpRequest) -> func.HttpResponse:
logging.info('Python HTTP trigger function processed a request.')
start_time = datetime.datetime.now().replace(microsecond=0)
print('Sample start: {}'.format(start_time))
print()
# Create a Batch service client. We'll now be interacting with the Batch
# service in addition to Storage
credentials = batch_auth.SharedKeyCredentials(config._BATCH_ACCOUNT_NAME,
config._BATCH_ACCOUNT_KEY)
batch_client = batch.BatchServiceClient(
credentials, batch_url=config._BATCH_ACCOUNT_URL)
name = req.params.get('name')
if not name:
try:
req_body = req.get_json()
except ValueError:
pass
else:
name = req_body.get('name')
if name:
task_id = add_tasks(batch_client, config._JOB_ID, name)
return func.HttpResponse(f"Task created {task_id}!")
else:
return func.HttpResponse(
"Please pass a name on the query string or in the request body",
status_code=400)
def batch_data_service_factory(cli_ctx, kwargs):
import azure.batch._batch_service_client as batch
import azure.batch.batch_auth as batchauth
account_name = kwargs.pop('account_name', None)
account_key = kwargs.pop('account_key', None)
account_endpoint = kwargs.pop('account_endpoint', None)
kwargs.pop('yes', None)
credentials = None
if not account_key:
from azure.cli.core._profile import Profile
profile = Profile(cli_ctx=cli_ctx)
# in order to use AAD auth in cloud shell mode, we will use mgmt AAD token
# instead of Batch AAD token to auth
if in_cloud_console():
resource = cli_ctx.cloud.endpoints.active_directory_resource_id
else:
resource = cli_ctx.cloud.endpoints.batch_resource_id
credentials, _, _ = profile.get_login_credentials(resource=resource)
else:
# Verify all values are populated and display readable error
if not all([account_name, account_key, account_endpoint]):
raise ValueError(
'usage error: --account-name NAME --account-key KEY --account-endpoint ENDPOINT'
)
credentials = batchauth.SharedKeyCredentials(account_name, account_key)
if not (account_endpoint.startswith('https://')
or account_endpoint.startswith('http://')):
account_endpoint = 'https://' + account_endpoint
return batch.BatchServiceClient(credentials,
batch_url=account_endpoint.rstrip('/'))
def execute_sample(global_config, sample_config):
"""Executes the sample with the specified configurations.
:param global_config: The global configuration to use.
:type global_config: `configparser.ConfigParser`
:param sample_config: The sample specific configuration to use.
:type sample_config: `configparser.ConfigParser`
"""
credentials = batchauth.SharedKeyCredentials(batch_account_name,
batch_account_key)
#credentials = ServicePrincipalCredentials(
# client_id=aad_client_id,
# secret=aad_client_secret,
# tenant=aad_tenant_id,
# resource="https://batch.core.windows.net/"
#)
batch_client = batch.BatchServiceClient(credentials,
batch_url=batch_service_url)
# Retry 5 times -- default is 3
batch_client.config.retry_policy.retries = 5
block_blob_client = azureblob.BlockBlobService(
account_name=storage_account_name,
account_key=storage_account_key,
endpoint_suffix=storage_account_suffix)
job_id = common.helpers.generate_unique_resource_name(
"poolsandresourcefilesjob")
try:
create_pool(batch_client, block_blob_client, pool_id, pool_vm_size,
pool_vm_count)
submit_job_and_add_task(batch_client, block_blob_client, job_id,
pool_id, storage_account_name)
common.helpers.wait_for_tasks_to_complete(
batch_client, job_id, datetime.timedelta(minutes=25))
tasks = batch_client.task.list(job_id)
task_ids = [task.id for task in tasks]
common.helpers.print_task_output(batch_client, job_id, task_ids)
print("Completed job_id:" + job_id)
finally:
# clean up
if should_delete_container:
block_blob_client.delete_container(_RESOURCE_CONTAINER_NAME,
fail_not_exist=False)
if should_delete_job:
print("Deleting job: ", job_id)
batch_client.job.delete(job_id)
if should_delete_pool:
print("Deleting pool: ", pool_id)
batch_client.pool.delete(pool_id)
def execute_sample(global_config, sample_config):
"""Executes the sample with the specified configurations.
:param global_config: The global configuration to use.
:type global_config: `configparser.ConfigParser`
:param sample_config: The sample specific configuration to use.
:type sample_config: `configparser.ConfigParser`
"""
# Set up the configuration
batch_account_key = global_config.get('Batch', 'batchaccountkey')
batch_account_name = global_config.get('Batch', 'batchaccountname')
batch_service_url = global_config.get('Batch', 'batchserviceurl')
should_delete_job = sample_config.getboolean('DEFAULT', 'shoulddeletejob')
pool_vm_size = sample_config.get('DEFAULT', 'poolvmsize')
pool_vm_count = sample_config.getint('DEFAULT', 'poolvmcount')
# Print the settings we are running with
common.helpers.print_configuration(global_config)
common.helpers.print_configuration(sample_config)
credentials = batchauth.SharedKeyCredentials(batch_account_name,
batch_account_key)
batch_client = batch.BatchServiceClient(credentials,
batch_url=batch_service_url)
# Retry 5 times -- default is 3
batch_client.config.retry_policy.retries = 5
job_id = common.helpers.generate_unique_resource_name("HelloWorld")
try:
submit_job_and_add_task(batch_client, job_id, pool_vm_size,
pool_vm_count)
common.helpers.wait_for_tasks_to_complete(
batch_client, job_id, datetime.timedelta(minutes=25))
tasks = batch_client.task.list(job_id)
task_ids = [task.id for task in tasks]
common.helpers.print_task_output(batch_client, job_id, task_ids)
finally:
if should_delete_job:
print("Deleting job: ", job_id)
batch_client.job.delete(job_id)
def authenticate_batch(
self,
service_principal: bool = False,
tenant_id: str = None,
client_id: str = None,
secret: str = None,
):
"""Authenticate to Batch service using credential provided in config['BATCH'], and saves batch client to self.batch_client.
Returns
-------
azure.batch.BatchServiceClient
Authenticated Azure Batch Service client.
"""
batch_account_name = (
self.config["BATCH"]["ACCOUNT_NAME"].strip("'"), )
location = self.config["GROUP"]["LOCATION"].strip("'")
if service_principal:
RESOURCE = "https://batch.core.windows.net/"
BATCH_ACCOUNT_URL = "https://{0}.{1}.batch.azure.com".format(
batch_account_name, location)
credentials = ServicePrincipalCredentials(client_id=client_id,
secret=secret,
tenant=tenant_id,
resource=RESOURCE)
else:
credentials = batch_auth.SharedKeyCredentials(
self.config["BATCH"]["ACCOUNT_NAME"].strip("'"),
self.config["BATCH"]["ACCOUNT_KEY"].strip("'"),
)
self.batch_client = batch.BatchServiceClient(
credentials, self.config["BATCH"]["ACCOUNT_URL"].strip("'"))
return self.batch_client
os.path.join(sys.path[0], 'taskdata1.txt'),
os.path.join(sys.path[0], 'taskdata2.txt')
]
# Upload the data files.
input_files = [
upload_file_to_container(container_client, file_path)
for file_path in input_file_paths
]
# Create a Batch service client. We'll now be interacting with the Batch
# service in addition to Storage
credentials = batch_auth.SharedKeyCredentials(config._BATCH_ACCOUNT_NAME,
config._BATCH_ACCOUNT_KEY)
batch_client = batch.BatchServiceClient(
credentials, batch_url=config._BATCH_ACCOUNT_URL)
try:
# Create the pool that will contain the compute nodes that will execute the
# tasks.
create_pool(batch_client, config._POOL_ID)
# Create the job that will run the tasks.
create_job(batch_client, config._JOB_ID, config._POOL_ID)
# Add the tasks to the job.
add_tasks(batch_client, config._JOB_ID, input_files)
# Pause execution until tasks reach Completed state.
wait_for_tasks_to_complete(batch_client, config._JOB_ID,
datetime.timedelta(minutes=30))
# Upload the script/data files to Azure Storage
input_files = [
common.helpers.upload_file_to_container(
blob_client, input_container_name, file_path, timeout=120)
for file_path in input_file_paths]
# Main application command to execute multiinstance task on a group of
# nodes, eg. MPI.
application_cmdline = [
'$AZ_BATCH_TASK_WORKING_DIR/application-cmd {}'.format(_NUM_INSTANCES)]
# Create a Batch service client. We'll now be interacting with the Batch
# service in addition to Storage
credentials = batchauth.SharedKeyCredentials(
_BATCH_ACCOUNT_NAME, _BATCH_ACCOUNT_KEY)
batch_client = batch.BatchServiceClient(
credentials, base_url=_BATCH_ACCOUNT_URL)
# Create the pool that will contain the compute nodes that will execute the
# tasks. The resource files we pass in are used for configuring the pool's
# start task, which is executed each time a node first joins the pool (or
# is rebooted or re-imaged).
multi_task_helpers.create_pool_and_wait_for_vms(
batch_client, _POOL_ID, _NODE_OS_PUBLISHER, _NODE_OS_OFFER,
_NODE_OS_SKU, _POOL_VM_SIZE, _POOL_NODE_COUNT)
# Create the job that will run the tasks.
common.helpers.create_job(batch_client, _JOB_ID, _POOL_ID)
# Add the tasks to the job. We need to supply a container shared access
# signature (SAS) token for the tasks so that they can upload their output
# to Azure Storage.
def execute_sample(global_config, sample_config):
"""Executes the sample with the specified configurations.
:param global_config: The global configuration to use.
:type global_config: `configparser.ConfigParser`
:param sample_config: The sample specific configuration to use.
:type sample_config: `configparser.ConfigParser`
"""
# Set up the configuration
batch_account_key = global_config.get('Batch', 'batchaccountkey')
batch_account_name = global_config.get('Batch', 'batchaccountname')
batch_service_url = global_config.get('Batch', 'batchserviceurl')
storage_account_key = global_config.get('Storage', 'storageaccountkey')
storage_account_name = global_config.get('Storage', 'storageaccountname')
storage_account_suffix = global_config.get('Storage',
'storageaccountsuffix')
should_delete_container = sample_config.getboolean(
'DEFAULT', 'shoulddeletecontainer')
should_delete_job_schedule = sample_config.getboolean(
'DEFAULT', 'shoulddeletejobschedule')
pool_vm_size = sample_config.get('DEFAULT', 'poolvmsize')
pool_vm_count = sample_config.getint('DEFAULT', 'poolvmcount')
# Print the settings we are running with
common.helpers.print_configuration(global_config)
common.helpers.print_configuration(sample_config)
credentials = batchauth.SharedKeyCredentials(batch_account_name,
batch_account_key)
batch_client = batch.BatchServiceClient(credentials,
batch_url=batch_service_url)
block_blob_client = azureblob.BlockBlobService(
account_name=storage_account_name,
account_key=storage_account_key,
endpoint_suffix=storage_account_suffix)
batch_client.config.retry_policy.retries = 5
job_schedule_id = common.helpers.generate_unique_resource_name(
"JobScheduler")
try:
create_job_schedule(batch_client, job_schedule_id, pool_vm_size,
pool_vm_count, block_blob_client)
print("Start time: ", _START_TIME)
print("Delete time: ", _END_TIME)
recent_job = common.helpers.wait_for_job_under_job_schedule(
batch_client, job_schedule_id, datetime.timedelta(minutes=5))
common.helpers.wait_for_tasks_to_complete(
batch_client, recent_job, datetime.timedelta(minutes=25))
tasks = batch_client.task.list(recent_job)
task_ids = [task.id for task in tasks]
common.helpers.print_task_output(batch_client, recent_job, task_ids)
common.helpers.wait_for_job_schedule_to_complete(
batch_client, job_schedule_id,
_END_TIME + datetime.timedelta(minutes=10))
except batchmodels.BatchErrorException as e:
for x in e.error.values:
print("BatchErrorException: ", x)
finally:
if should_delete_job_schedule:
print("Deleting job schedule: ", job_schedule_id)
batch_client.job_schedule.delete(job_schedule_id)
if should_delete_container:
block_blob_client.delete_container(_CONTAINER_NAME,
fail_not_exist=False)
def execute_sample(global_config, sample_config):
"""Executes the sample with the specified configurations.
:param global_config: The global configuration to use.
:type global_config: `configparser.ConfigParser`
:param sample_config: The sample specific configuration to use.
:type sample_config: `configparser.ConfigParser`
"""
# Set up the configuration
batch_account_key = global_config.get('Batch', 'batchaccountkey')
batch_account_name = global_config.get('Batch', 'batchaccountname')
batch_service_url = global_config.get('Batch', 'batchserviceurl')
storage_account_key = global_config.get('Storage', 'storageaccountkey')
storage_account_name = global_config.get('Storage', 'storageaccountname')
storage_account_suffix = global_config.get('Storage',
'storageaccountsuffix')
should_delete_container = sample_config.getboolean(
'DEFAULT', 'shoulddeletecontainer')
should_delete_job = sample_config.getboolean('DEFAULT', 'shoulddeletejob')
should_delete_pool = sample_config.getboolean('DEFAULT',
'shoulddeletepool')
should_delete_cert = sample_config.getboolean('DEFAULT',
'shoulddeletecert')
pool_vm_size = sample_config.get('DEFAULT', 'poolvmsize')
pool_vm_count = sample_config.getint('DEFAULT', 'poolvmcount')
# Print the settings we are running with
common.helpers.print_configuration(global_config)
common.helpers.print_configuration(sample_config)
credentials = batchauth.SharedKeyCredentials(batch_account_name,
batch_account_key)
batch_client = batch.BatchServiceClient(credentials,
batch_url=batch_service_url)
# Retry 5 times -- default is 3
batch_client.config.retry_policy.retries = 5
block_blob_client = azureblob.BlockBlobService(
account_name=storage_account_name,
account_key=storage_account_key,
endpoint_suffix=storage_account_suffix)
job_id = common.helpers.generate_unique_resource_name(
'EncryptedResourceFiles')
pool_id = common.helpers.generate_unique_resource_name(
'EncryptedResourceFiles')
sha1_cert_tp = None
try:
# encrypt local file and upload to blob storage via blobxfer
rsapfxfile, sha1_cert_tp = encrypt_localfile_to_blob_storage(
storage_account_name, storage_account_key, _CONTAINER_NAME,
_RESOURCE_TO_ENCRYPT)
# add certificate to account
add_certificate_to_account(batch_client, rsapfxfile, _PFX_PASSPHRASE,
sha1_cert_tp)
# create pool and wait for node idle
create_pool_and_wait_for_node(batch_client, pool_id, pool_vm_size,
pool_vm_count, sha1_cert_tp)
# submit job and add a task
submit_job_and_add_task(batch_client, block_blob_client,
storage_account_name, storage_account_key,
_CONTAINER_NAME, _RESOURCE_NAME, job_id,
pool_id, sha1_cert_tp)
# wait for tasks to complete
common.helpers.wait_for_tasks_to_complete(
batch_client, job_id, datetime.timedelta(minutes=20))
tasks = batch_client.task.list(job_id)
task_ids = [task.id for task in tasks]
common.helpers.print_task_output(batch_client, job_id, task_ids)
finally:
# perform clean up
if should_delete_container:
print('Deleting container: {}'.format(_CONTAINER_NAME))
block_blob_client.delete_container(_CONTAINER_NAME,
fail_not_exist=False)
if should_delete_job:
print('Deleting job: {}'.format(job_id))
batch_client.job.delete(job_id)
if should_delete_pool:
print('Deleting pool: {}'.format(pool_id))
batch_client.pool.delete(pool_id)
if should_delete_cert and sha1_cert_tp is not None:
# cert deletion requires no active references to cert, so
# override any config settings for preserving job/pool
if not should_delete_job:
print('Deleting job: {}'.format(job_id))
batch_client.job.delete(job_id)
if not should_delete_pool:
print('Deleting pool: {}'.format(pool_id))
batch_client.pool.delete(pool_id)
print('Deleting cert: {}'.format(sha1_cert_tp))
batch_client.certificate.delete('sha1', sha1_cert_tp)
def execute_sample(global_config, sample_config):
"""Executes the sample with the specified configurations.
:param global_config: The global configuration to use.
:type global_config: `configparser.ConfigParser`
:param sample_config: The sample specific configuration to use.
:type sample_config: `configparser.ConfigParser`
"""
# Set up the configuration
batch_account_key = global_config.get('Batch', 'batchaccountkey')
batch_account_name = global_config.get('Batch', 'batchaccountname')
batch_service_url = global_config.get('Batch', 'batchserviceurl')
storage_account_key = global_config.get('Storage', 'storageaccountkey')
storage_account_name = global_config.get('Storage', 'storageaccountname')
storage_account_suffix = global_config.get('Storage',
'storageaccountsuffix')
should_delete_container = sample_config.getboolean(
'DEFAULT', 'shoulddeletecontainer')
should_delete_job = sample_config.getboolean('DEFAULT', 'shoulddeletejob')
should_delete_pool = sample_config.getboolean('DEFAULT',
'shoulddeletepool')
pool_vm_size = sample_config.get('DEFAULT', 'poolvmsize')
pool_vm_count = sample_config.getint('DEFAULT', 'poolvmcount')
# Print the settings we are running with
common.helpers.print_configuration(global_config)
common.helpers.print_configuration(sample_config)
credentials = batchauth.SharedKeyCredentials(batch_account_name,
batch_account_key)
batch_client = batch.BatchServiceClient(credentials,
batch_url=batch_service_url)
# Retry 5 times -- default is 3
batch_client.config.retry_policy.retries = 5
block_blob_client = azureblob.BlockBlobService(
account_name=storage_account_name,
account_key=storage_account_key,
endpoint_suffix=storage_account_suffix)
job_id = common.helpers.generate_unique_resource_name(
"PoolsAndResourceFilesJob")
pool_id = "PoolsAndResourceFilesPool"
try:
create_pool(batch_client, block_blob_client, pool_id, pool_vm_size,
pool_vm_count)
submit_job_and_add_task(batch_client, block_blob_client, job_id,
pool_id)
common.helpers.wait_for_tasks_to_complete(
batch_client, job_id, datetime.timedelta(minutes=25))
tasks = batch_client.task.list(job_id)
task_ids = [task.id for task in tasks]
common.helpers.print_task_output(batch_client, job_id, task_ids)
finally:
# clean up
if should_delete_container:
block_blob_client.delete_container(_CONTAINER_NAME,
fail_not_exist=False)
if should_delete_job:
print("Deleting job: ", job_id)
batch_client.job.delete(job_id)
if should_delete_pool:
print("Deleting pool: ", pool_id)
batch_client.pool.delete(pool_id)
def execute_sample(global_config, sample_config):
"""Executes the sample with the specified configurations.
:param global_config: The global configuration to use.
:type global_config: `configparser.ConfigParser`
:param sample_config: The sample specific configuration to use.
:type sample_config: `configparser.ConfigParser`
"""
# Set up the configuration
batch_account_key = global_config.get('Batch', 'batchaccountkey')
batch_account_name = global_config.get('Batch', 'batchaccountname')
batch_service_url = global_config.get('Batch', 'batchserviceurl')
storage_account_key = global_config.get('Storage', 'storageaccountkey')
storage_account_name = global_config.get('Storage', 'storageaccountname')
storage_account_suffix = global_config.get('Storage',
'storageaccountsuffix')
should_delete_container = sample_config.getboolean(
'DEFAULT', 'shoulddeletecontainer')
should_delete_job = sample_config.getboolean('DEFAULT', 'shoulddeletejob')
should_delete_pool = sample_config.getboolean('DEFAULT',
'shoulddeletepool')
pool_vm_size = sample_config.get('DEFAULT', 'poolvmsize')
pool_vm_count = sample_config.getint('DEFAULT', 'poolvmcount')
blockfacefile = sample_config.get('BLOCKS', 'blockfacefile')
blockstart = sample_config.getint('BLOCKS', 'startblockindex')
blockend = sample_config.getint('BLOCKS', 'endblockindex')
block_inds = pd.read_csv(
blockfacefile).sourceelementkey.values[blockstart:blockend]
block_indices = []
for block in block_inds:
if not os.path.isfile('output/%d_model.dat' % block):
block_indices.append(block)
# Print the settings we are running with
common.helpers.print_configuration(global_config)
common.helpers.print_configuration(sample_config)
credentials = batchauth.SharedKeyCredentials(batch_account_name,
batch_account_key)
batch_client = batch.BatchServiceClient(credentials,
batch_url=batch_service_url)
# Retry 5 times -- default is 3
batch_client.config.retry_policy.retries = 5
block_blob_client = azureblob.BlockBlobService(
account_name=storage_account_name,
account_key=storage_account_key,
endpoint_suffix=storage_account_suffix)
# Use the blob client to create the containers in Azure Storage if they
# don't yet exist.
input_container_name = 'input'
output_container_name = 'output'
block_blob_client.create_container(input_container_name,
fail_on_exist=False)
block_blob_client.create_container(output_container_name,
fail_on_exist=False)
print('Container [{}] created.'.format(input_container_name))
print('Container [{}] created.'.format(output_container_name))
# generate list of input csv files
input_file_paths = create_csv_list(block_indices)
# Upload the input files. This is the collection of files that are to be processed by the tasks.
input_files = [
common.helpers.upload_file_to_container(block_blob_client,
input_container_name,
file_path, _EXPIRY_TIME, 60)
for file_path in input_file_paths
]
# Obtain a shared access signature URL that provides write access to the output
# container to which the tasks will upload their output.
output_container_sas_url = common.helpers.get_container_sas_url(
block_blob_client, output_container_name,
azureblob.BlobPermissions.WRITE, output_container_name)
job_id = common.helpers.generate_unique_resource_name(
"PoolsAndResourceFilesJob")
pool_id = common.helpers.generate_unique_resource_name("ArimaPool")
try:
create_pool(batch_client, block_blob_client, pool_id, pool_vm_size,
pool_vm_count)
submit_job_and_add_task(batch_client, block_blob_client, job_id,
pool_id, block_indices, input_files,
output_container_sas_url)
common.helpers.wait_for_tasks_to_complete(batch_client, job_id,
datetime.timedelta(hours=72))
tasks = batch_client.task.list(job_id)
task_ids = [task.id for task in tasks]
common.helpers.print_task_output(batch_client, job_id, task_ids)
finally:
# clean up
if should_delete_container:
block_blob_client.delete_container(_CONTAINER_NAME,
fail_not_exist=False)
if should_delete_job:
print("Deleting job: ", job_id)
batch_client.job.delete(job_id)
if should_delete_pool:
print("Deleting pool: ", pool_id)
batch_client.pool.delete(pool_id)
