def main(req: func.HttpRequest) -> func.HttpResponse: logging.info('Python HTTP trigger function processed a request.') start_time = datetime.datetime.now().replace(microsecond=0) print('Sample start: {}'.format(start_time)) print() # Create a Batch service client. We'll now be interacting with the Batch # service in addition to Storage credentials = batch_auth.SharedKeyCredentials(config._BATCH_ACCOUNT_NAME, config._BATCH_ACCOUNT_KEY) batch_client = batch.BatchServiceClient( credentials, batch_url=config._BATCH_ACCOUNT_URL) name = req.params.get('name') if not name: try: req_body = req.get_json() except ValueError: pass else: name = req_body.get('name') if name: task_id = add_tasks(batch_client, config._JOB_ID, name) return func.HttpResponse(f"Task created {task_id}!") else: return func.HttpResponse( "Please pass a name on the query string or in the request body", status_code=400)
def batch_data_service_factory(cli_ctx, kwargs): import azure.batch._batch_service_client as batch import azure.batch.batch_auth as batchauth account_name = kwargs.pop('account_name', None) account_key = kwargs.pop('account_key', None) account_endpoint = kwargs.pop('account_endpoint', None) kwargs.pop('yes', None) credentials = None if not account_key: from azure.cli.core._profile import Profile profile = Profile(cli_ctx=cli_ctx) # in order to use AAD auth in cloud shell mode, we will use mgmt AAD token # instead of Batch AAD token to auth if in_cloud_console(): resource = cli_ctx.cloud.endpoints.active_directory_resource_id else: resource = cli_ctx.cloud.endpoints.batch_resource_id credentials, _, _ = profile.get_login_credentials(resource=resource) else: # Verify all values are populated and display readable error if not all([account_name, account_key, account_endpoint]): raise ValueError( 'usage error: --account-name NAME --account-key KEY --account-endpoint ENDPOINT' ) credentials = batchauth.SharedKeyCredentials(account_name, account_key) if not (account_endpoint.startswith('https://') or account_endpoint.startswith('http://')): account_endpoint = 'https://' + account_endpoint return batch.BatchServiceClient(credentials, batch_url=account_endpoint.rstrip('/'))
def execute_sample(global_config, sample_config): """Executes the sample with the specified configurations. :param global_config: The global configuration to use. :type global_config: `configparser.ConfigParser` :param sample_config: The sample specific configuration to use. :type sample_config: `configparser.ConfigParser` """ credentials = batchauth.SharedKeyCredentials(batch_account_name, batch_account_key) #credentials = ServicePrincipalCredentials( # client_id=aad_client_id, # secret=aad_client_secret, # tenant=aad_tenant_id, # resource="https://batch.core.windows.net/" #) batch_client = batch.BatchServiceClient(credentials, batch_url=batch_service_url) # Retry 5 times -- default is 3 batch_client.config.retry_policy.retries = 5 block_blob_client = azureblob.BlockBlobService( account_name=storage_account_name, account_key=storage_account_key, endpoint_suffix=storage_account_suffix) job_id = common.helpers.generate_unique_resource_name( "poolsandresourcefilesjob") try: create_pool(batch_client, block_blob_client, pool_id, pool_vm_size, pool_vm_count) submit_job_and_add_task(batch_client, block_blob_client, job_id, pool_id, storage_account_name) common.helpers.wait_for_tasks_to_complete( batch_client, job_id, datetime.timedelta(minutes=25)) tasks = batch_client.task.list(job_id) task_ids = [task.id for task in tasks] common.helpers.print_task_output(batch_client, job_id, task_ids) print("Completed job_id:" + job_id) finally: # clean up if should_delete_container: block_blob_client.delete_container(_RESOURCE_CONTAINER_NAME, fail_not_exist=False) if should_delete_job: print("Deleting job: ", job_id) batch_client.job.delete(job_id) if should_delete_pool: print("Deleting pool: ", pool_id) batch_client.pool.delete(pool_id)
def execute_sample(global_config, sample_config): """Executes the sample with the specified configurations. :param global_config: The global configuration to use. :type global_config: `configparser.ConfigParser` :param sample_config: The sample specific configuration to use. :type sample_config: `configparser.ConfigParser` """ # Set up the configuration batch_account_key = global_config.get('Batch', 'batchaccountkey') batch_account_name = global_config.get('Batch', 'batchaccountname') batch_service_url = global_config.get('Batch', 'batchserviceurl') should_delete_job = sample_config.getboolean('DEFAULT', 'shoulddeletejob') pool_vm_size = sample_config.get('DEFAULT', 'poolvmsize') pool_vm_count = sample_config.getint('DEFAULT', 'poolvmcount') # Print the settings we are running with common.helpers.print_configuration(global_config) common.helpers.print_configuration(sample_config) credentials = batchauth.SharedKeyCredentials(batch_account_name, batch_account_key) batch_client = batch.BatchServiceClient(credentials, batch_url=batch_service_url) # Retry 5 times -- default is 3 batch_client.config.retry_policy.retries = 5 job_id = common.helpers.generate_unique_resource_name("HelloWorld") try: submit_job_and_add_task(batch_client, job_id, pool_vm_size, pool_vm_count) common.helpers.wait_for_tasks_to_complete( batch_client, job_id, datetime.timedelta(minutes=25)) tasks = batch_client.task.list(job_id) task_ids = [task.id for task in tasks] common.helpers.print_task_output(batch_client, job_id, task_ids) finally: if should_delete_job: print("Deleting job: ", job_id) batch_client.job.delete(job_id)
def authenticate_batch( self, service_principal: bool = False, tenant_id: str = None, client_id: str = None, secret: str = None, ): """Authenticate to Batch service using credential provided in config['BATCH'], and saves batch client to self.batch_client. Returns ------- azure.batch.BatchServiceClient Authenticated Azure Batch Service client. """ batch_account_name = ( self.config["BATCH"]["ACCOUNT_NAME"].strip("'"), ) location = self.config["GROUP"]["LOCATION"].strip("'") if service_principal: RESOURCE = "https://batch.core.windows.net/" BATCH_ACCOUNT_URL = "https://{0}.{1}.batch.azure.com".format( batch_account_name, location) credentials = ServicePrincipalCredentials(client_id=client_id, secret=secret, tenant=tenant_id, resource=RESOURCE) else: credentials = batch_auth.SharedKeyCredentials( self.config["BATCH"]["ACCOUNT_NAME"].strip("'"), self.config["BATCH"]["ACCOUNT_KEY"].strip("'"), ) self.batch_client = batch.BatchServiceClient( credentials, self.config["BATCH"]["ACCOUNT_URL"].strip("'")) return self.batch_client
os.path.join(sys.path[0], 'taskdata1.txt'), os.path.join(sys.path[0], 'taskdata2.txt') ] # Upload the data files. input_files = [ upload_file_to_container(container_client, file_path) for file_path in input_file_paths ] # Create a Batch service client. We'll now be interacting with the Batch # service in addition to Storage credentials = batch_auth.SharedKeyCredentials(config._BATCH_ACCOUNT_NAME, config._BATCH_ACCOUNT_KEY) batch_client = batch.BatchServiceClient( credentials, batch_url=config._BATCH_ACCOUNT_URL) try: # Create the pool that will contain the compute nodes that will execute the # tasks. create_pool(batch_client, config._POOL_ID) # Create the job that will run the tasks. create_job(batch_client, config._JOB_ID, config._POOL_ID) # Add the tasks to the job. add_tasks(batch_client, config._JOB_ID, input_files) # Pause execution until tasks reach Completed state. wait_for_tasks_to_complete(batch_client, config._JOB_ID, datetime.timedelta(minutes=30))
# Upload the script/data files to Azure Storage input_files = [ common.helpers.upload_file_to_container( blob_client, input_container_name, file_path, timeout=120) for file_path in input_file_paths] # Main application command to execute multiinstance task on a group of # nodes, eg. MPI. application_cmdline = [ '$AZ_BATCH_TASK_WORKING_DIR/application-cmd {}'.format(_NUM_INSTANCES)] # Create a Batch service client. We'll now be interacting with the Batch # service in addition to Storage credentials = batchauth.SharedKeyCredentials( _BATCH_ACCOUNT_NAME, _BATCH_ACCOUNT_KEY) batch_client = batch.BatchServiceClient( credentials, base_url=_BATCH_ACCOUNT_URL) # Create the pool that will contain the compute nodes that will execute the # tasks. The resource files we pass in are used for configuring the pool's # start task, which is executed each time a node first joins the pool (or # is rebooted or re-imaged). multi_task_helpers.create_pool_and_wait_for_vms( batch_client, _POOL_ID, _NODE_OS_PUBLISHER, _NODE_OS_OFFER, _NODE_OS_SKU, _POOL_VM_SIZE, _POOL_NODE_COUNT) # Create the job that will run the tasks. common.helpers.create_job(batch_client, _JOB_ID, _POOL_ID) # Add the tasks to the job. We need to supply a container shared access # signature (SAS) token for the tasks so that they can upload their output # to Azure Storage.
def execute_sample(global_config, sample_config): """Executes the sample with the specified configurations. :param global_config: The global configuration to use. :type global_config: `configparser.ConfigParser` :param sample_config: The sample specific configuration to use. :type sample_config: `configparser.ConfigParser` """ # Set up the configuration batch_account_key = global_config.get('Batch', 'batchaccountkey') batch_account_name = global_config.get('Batch', 'batchaccountname') batch_service_url = global_config.get('Batch', 'batchserviceurl') storage_account_key = global_config.get('Storage', 'storageaccountkey') storage_account_name = global_config.get('Storage', 'storageaccountname') storage_account_suffix = global_config.get('Storage', 'storageaccountsuffix') should_delete_container = sample_config.getboolean( 'DEFAULT', 'shoulddeletecontainer') should_delete_job_schedule = sample_config.getboolean( 'DEFAULT', 'shoulddeletejobschedule') pool_vm_size = sample_config.get('DEFAULT', 'poolvmsize') pool_vm_count = sample_config.getint('DEFAULT', 'poolvmcount') # Print the settings we are running with common.helpers.print_configuration(global_config) common.helpers.print_configuration(sample_config) credentials = batchauth.SharedKeyCredentials(batch_account_name, batch_account_key) batch_client = batch.BatchServiceClient(credentials, batch_url=batch_service_url) block_blob_client = azureblob.BlockBlobService( account_name=storage_account_name, account_key=storage_account_key, endpoint_suffix=storage_account_suffix) batch_client.config.retry_policy.retries = 5 job_schedule_id = common.helpers.generate_unique_resource_name( "JobScheduler") try: create_job_schedule(batch_client, job_schedule_id, pool_vm_size, pool_vm_count, block_blob_client) print("Start time: ", _START_TIME) print("Delete time: ", _END_TIME) recent_job = common.helpers.wait_for_job_under_job_schedule( batch_client, job_schedule_id, datetime.timedelta(minutes=5)) common.helpers.wait_for_tasks_to_complete( batch_client, recent_job, datetime.timedelta(minutes=25)) tasks = batch_client.task.list(recent_job) task_ids = [task.id for task in tasks] common.helpers.print_task_output(batch_client, recent_job, task_ids) common.helpers.wait_for_job_schedule_to_complete( batch_client, job_schedule_id, _END_TIME + datetime.timedelta(minutes=10)) except batchmodels.BatchErrorException as e: for x in e.error.values: print("BatchErrorException: ", x) finally: if should_delete_job_schedule: print("Deleting job schedule: ", job_schedule_id) batch_client.job_schedule.delete(job_schedule_id) if should_delete_container: block_blob_client.delete_container(_CONTAINER_NAME, fail_not_exist=False)
def execute_sample(global_config, sample_config): """Executes the sample with the specified configurations. :param global_config: The global configuration to use. :type global_config: `configparser.ConfigParser` :param sample_config: The sample specific configuration to use. :type sample_config: `configparser.ConfigParser` """ # Set up the configuration batch_account_key = global_config.get('Batch', 'batchaccountkey') batch_account_name = global_config.get('Batch', 'batchaccountname') batch_service_url = global_config.get('Batch', 'batchserviceurl') storage_account_key = global_config.get('Storage', 'storageaccountkey') storage_account_name = global_config.get('Storage', 'storageaccountname') storage_account_suffix = global_config.get('Storage', 'storageaccountsuffix') should_delete_container = sample_config.getboolean( 'DEFAULT', 'shoulddeletecontainer') should_delete_job = sample_config.getboolean('DEFAULT', 'shoulddeletejob') should_delete_pool = sample_config.getboolean('DEFAULT', 'shoulddeletepool') should_delete_cert = sample_config.getboolean('DEFAULT', 'shoulddeletecert') pool_vm_size = sample_config.get('DEFAULT', 'poolvmsize') pool_vm_count = sample_config.getint('DEFAULT', 'poolvmcount') # Print the settings we are running with common.helpers.print_configuration(global_config) common.helpers.print_configuration(sample_config) credentials = batchauth.SharedKeyCredentials(batch_account_name, batch_account_key) batch_client = batch.BatchServiceClient(credentials, batch_url=batch_service_url) # Retry 5 times -- default is 3 batch_client.config.retry_policy.retries = 5 block_blob_client = azureblob.BlockBlobService( account_name=storage_account_name, account_key=storage_account_key, endpoint_suffix=storage_account_suffix) job_id = common.helpers.generate_unique_resource_name( 'EncryptedResourceFiles') pool_id = common.helpers.generate_unique_resource_name( 'EncryptedResourceFiles') sha1_cert_tp = None try: # encrypt local file and upload to blob storage via blobxfer rsapfxfile, sha1_cert_tp = encrypt_localfile_to_blob_storage( storage_account_name, storage_account_key, _CONTAINER_NAME, _RESOURCE_TO_ENCRYPT) # add certificate to account add_certificate_to_account(batch_client, rsapfxfile, _PFX_PASSPHRASE, sha1_cert_tp) # create pool and wait for node idle create_pool_and_wait_for_node(batch_client, pool_id, pool_vm_size, pool_vm_count, sha1_cert_tp) # submit job and add a task submit_job_and_add_task(batch_client, block_blob_client, storage_account_name, storage_account_key, _CONTAINER_NAME, _RESOURCE_NAME, job_id, pool_id, sha1_cert_tp) # wait for tasks to complete common.helpers.wait_for_tasks_to_complete( batch_client, job_id, datetime.timedelta(minutes=20)) tasks = batch_client.task.list(job_id) task_ids = [task.id for task in tasks] common.helpers.print_task_output(batch_client, job_id, task_ids) finally: # perform clean up if should_delete_container: print('Deleting container: {}'.format(_CONTAINER_NAME)) block_blob_client.delete_container(_CONTAINER_NAME, fail_not_exist=False) if should_delete_job: print('Deleting job: {}'.format(job_id)) batch_client.job.delete(job_id) if should_delete_pool: print('Deleting pool: {}'.format(pool_id)) batch_client.pool.delete(pool_id) if should_delete_cert and sha1_cert_tp is not None: # cert deletion requires no active references to cert, so # override any config settings for preserving job/pool if not should_delete_job: print('Deleting job: {}'.format(job_id)) batch_client.job.delete(job_id) if not should_delete_pool: print('Deleting pool: {}'.format(pool_id)) batch_client.pool.delete(pool_id) print('Deleting cert: {}'.format(sha1_cert_tp)) batch_client.certificate.delete('sha1', sha1_cert_tp)
def execute_sample(global_config, sample_config): """Executes the sample with the specified configurations. :param global_config: The global configuration to use. :type global_config: `configparser.ConfigParser` :param sample_config: The sample specific configuration to use. :type sample_config: `configparser.ConfigParser` """ # Set up the configuration batch_account_key = global_config.get('Batch', 'batchaccountkey') batch_account_name = global_config.get('Batch', 'batchaccountname') batch_service_url = global_config.get('Batch', 'batchserviceurl') storage_account_key = global_config.get('Storage', 'storageaccountkey') storage_account_name = global_config.get('Storage', 'storageaccountname') storage_account_suffix = global_config.get('Storage', 'storageaccountsuffix') should_delete_container = sample_config.getboolean( 'DEFAULT', 'shoulddeletecontainer') should_delete_job = sample_config.getboolean('DEFAULT', 'shoulddeletejob') should_delete_pool = sample_config.getboolean('DEFAULT', 'shoulddeletepool') pool_vm_size = sample_config.get('DEFAULT', 'poolvmsize') pool_vm_count = sample_config.getint('DEFAULT', 'poolvmcount') # Print the settings we are running with common.helpers.print_configuration(global_config) common.helpers.print_configuration(sample_config) credentials = batchauth.SharedKeyCredentials(batch_account_name, batch_account_key) batch_client = batch.BatchServiceClient(credentials, batch_url=batch_service_url) # Retry 5 times -- default is 3 batch_client.config.retry_policy.retries = 5 block_blob_client = azureblob.BlockBlobService( account_name=storage_account_name, account_key=storage_account_key, endpoint_suffix=storage_account_suffix) job_id = common.helpers.generate_unique_resource_name( "PoolsAndResourceFilesJob") pool_id = "PoolsAndResourceFilesPool" try: create_pool(batch_client, block_blob_client, pool_id, pool_vm_size, pool_vm_count) submit_job_and_add_task(batch_client, block_blob_client, job_id, pool_id) common.helpers.wait_for_tasks_to_complete( batch_client, job_id, datetime.timedelta(minutes=25)) tasks = batch_client.task.list(job_id) task_ids = [task.id for task in tasks] common.helpers.print_task_output(batch_client, job_id, task_ids) finally: # clean up if should_delete_container: block_blob_client.delete_container(_CONTAINER_NAME, fail_not_exist=False) if should_delete_job: print("Deleting job: ", job_id) batch_client.job.delete(job_id) if should_delete_pool: print("Deleting pool: ", pool_id) batch_client.pool.delete(pool_id)
def execute_sample(global_config, sample_config): """Executes the sample with the specified configurations. :param global_config: The global configuration to use. :type global_config: `configparser.ConfigParser` :param sample_config: The sample specific configuration to use. :type sample_config: `configparser.ConfigParser` """ # Set up the configuration batch_account_key = global_config.get('Batch', 'batchaccountkey') batch_account_name = global_config.get('Batch', 'batchaccountname') batch_service_url = global_config.get('Batch', 'batchserviceurl') storage_account_key = global_config.get('Storage', 'storageaccountkey') storage_account_name = global_config.get('Storage', 'storageaccountname') storage_account_suffix = global_config.get('Storage', 'storageaccountsuffix') should_delete_container = sample_config.getboolean( 'DEFAULT', 'shoulddeletecontainer') should_delete_job = sample_config.getboolean('DEFAULT', 'shoulddeletejob') should_delete_pool = sample_config.getboolean('DEFAULT', 'shoulddeletepool') pool_vm_size = sample_config.get('DEFAULT', 'poolvmsize') pool_vm_count = sample_config.getint('DEFAULT', 'poolvmcount') blockfacefile = sample_config.get('BLOCKS', 'blockfacefile') blockstart = sample_config.getint('BLOCKS', 'startblockindex') blockend = sample_config.getint('BLOCKS', 'endblockindex') block_inds = pd.read_csv( blockfacefile).sourceelementkey.values[blockstart:blockend] block_indices = [] for block in block_inds: if not os.path.isfile('output/%d_model.dat' % block): block_indices.append(block) # Print the settings we are running with common.helpers.print_configuration(global_config) common.helpers.print_configuration(sample_config) credentials = batchauth.SharedKeyCredentials(batch_account_name, batch_account_key) batch_client = batch.BatchServiceClient(credentials, batch_url=batch_service_url) # Retry 5 times -- default is 3 batch_client.config.retry_policy.retries = 5 block_blob_client = azureblob.BlockBlobService( account_name=storage_account_name, account_key=storage_account_key, endpoint_suffix=storage_account_suffix) # Use the blob client to create the containers in Azure Storage if they # don't yet exist. input_container_name = 'input' output_container_name = 'output' block_blob_client.create_container(input_container_name, fail_on_exist=False) block_blob_client.create_container(output_container_name, fail_on_exist=False) print('Container [{}] created.'.format(input_container_name)) print('Container [{}] created.'.format(output_container_name)) # generate list of input csv files input_file_paths = create_csv_list(block_indices) # Upload the input files. This is the collection of files that are to be processed by the tasks. input_files = [ common.helpers.upload_file_to_container(block_blob_client, input_container_name, file_path, _EXPIRY_TIME, 60) for file_path in input_file_paths ] # Obtain a shared access signature URL that provides write access to the output # container to which the tasks will upload their output. output_container_sas_url = common.helpers.get_container_sas_url( block_blob_client, output_container_name, azureblob.BlobPermissions.WRITE, output_container_name) job_id = common.helpers.generate_unique_resource_name( "PoolsAndResourceFilesJob") pool_id = common.helpers.generate_unique_resource_name("ArimaPool") try: create_pool(batch_client, block_blob_client, pool_id, pool_vm_size, pool_vm_count) submit_job_and_add_task(batch_client, block_blob_client, job_id, pool_id, block_indices, input_files, output_container_sas_url) common.helpers.wait_for_tasks_to_complete(batch_client, job_id, datetime.timedelta(hours=72)) tasks = batch_client.task.list(job_id) task_ids = [task.id for task in tasks] common.helpers.print_task_output(batch_client, job_id, task_ids) finally: # clean up if should_delete_container: block_blob_client.delete_container(_CONTAINER_NAME, fail_not_exist=False) if should_delete_job: print("Deleting job: ", job_id) batch_client.job.delete(job_id) if should_delete_pool: print("Deleting pool: ", pool_id) batch_client.pool.delete(pool_id)