def create_job(client: super_batch.Client, batch_dir: str) -> None: r""" :param client: A :class:`super_batch.Client` instance with the Azure Batch run parameters :type client: :class:super_batch.Client :param str batch_dir: path of the local batch temp directory """ _LOCAL_INPUT_FILE = join(batch_dir, _BATCH_CV_FILE_NAME) v_pen, w_pen, model_data = get_config(_LOCAL_INPUT_FILE) n_folds = len(model_data["folds"]) * len(v_pen) * len(w_pen) # CREATE THE COMMON IMPUT FILE RESOURCE input_resource = client.build_resource_file(_LOCAL_INPUT_FILE, _CONTAINER_INPUT_FILE) for fold_number in range(n_folds): # BUILD THE COMMAND LINE command_line = "/bin/bash -c 'stt {} {} {}'".format( _CONTAINER_INPUT_FILE, _CONTAINER_OUTPUT_FILE, fold_number) # CREATE AN OUTPUT RESOURCE: output_resource = client.build_output_file( _CONTAINER_OUTPUT_FILE, LOCAL_OUTPUTS_PATTERN.format(fold_number)) # CREATE A TASK client.add_task([input_resource], [output_resource], command_line=command_line)
def load_results(config: BatchConfig) -> None: r""" :param config: A :class:`BatchConfig` instance with the Azure Batch run parameters :type config: :class:BatchConfig :raises BatchErrorException: If raised by the Azure Batch Python SDK """ # pylint: disable=too-many-locals # replace any missing values in the configuration with environment variables config = validate_config(config) start_time = datetime.datetime.now().replace(microsecond=0) print('Load result for job "{}" start time: {}'.format( config.JOB_ID, start_time)) print() _LOCAL_INPUT_FILE = os.path.join(config.BATCH_DIRECTORY, _BATCH_CV_FILE_NAME) v_pen, w_pen, model_data = get_config(_LOCAL_INPUT_FILE) n_folds = len(model_data["folds"]) * len(v_pen) * len(w_pen) # Create the blob client, for use in obtaining references to # blob storage containers and uploading files to containers. blob_client = azureblob.BlockBlobService( account_name=config.STORAGE_ACCOUNT_NAME, account_key=config.STORAGE_ACCOUNT_KEY) # Create a Batch service client. We'll now be interacting with the Batch # service in addition to Storage credentials = batch_auth.SharedKeyCredentials(config.BATCH_ACCOUNT_NAME, config.BATCH_ACCOUNT_KEY) batch_client = batch.BatchServiceClient(credentials, batch_url=config.BATCH_ACCOUNT_URL) try: # Pause execution until tasks reach Completed state. wait_for_tasks_to_complete( batch_client, config.JOB_ID, datetime.timedelta(hours=config.STORAGE_ACCESS_DURATION_HRS)) _download_files(config, blob_client, config.BATCH_DIRECTORY, n_folds) except models.BatchErrorException as err: print_batch_exception(err) raise err # Clean up storage resources # TODO: re-enable this and delete the output container too # -- print("Deleting container [{}]...".format(input_container_name)) # -- blob_client.delete_container(input_container_name) # Print out some timing info end_time = datetime.datetime.now().replace(microsecond=0) print() print("Sample end: {}".format(end_time)) print("Elapsed time: {}".format(end_time - start_time)) print() # Clean up Batch resources (if the user so chooses). if config.DELETE_POOL_WHEN_DONE: batch_client.pool.delete(config.POOL_ID) if config.DELETE_JOB_WHEN_DONE: batch_client.job.delete(config.JOB_ID)
def run(config: BatchConfig, wait=True) -> None: r""" :param config: A :class:`BatchConfig` instance with the Azure Batch run parameters :type config: :class:BatchConfig :param boolean wait: If true, wait for the batch to complete and then download the results to file :raises BatchErrorException: If raised by the Azure Batch Python SDK """ # pylint: disable=too-many-locals # replace any missing values in the configuration with environment variables config = validate_config(config) start_time = datetime.datetime.now().replace(microsecond=0) print('Synthetic Controls Run "{}" start time: {}'.format( config.JOB_ID, start_time)) print() _LOCAL_INPUT_FILE = os.path.join(config.BATCH_DIRECTORY, _BATCH_CV_FILE_NAME) v_pen, w_pen, model_data = get_config(_LOCAL_INPUT_FILE) n_folds = len(model_data["folds"]) * len(v_pen) * len(w_pen) # Create the blob client, for use in obtaining references to # blob storage containers and uploading files to containers. blob_client = azureblob.BlockBlobService( account_name=config.STORAGE_ACCOUNT_NAME, account_key=config.STORAGE_ACCOUNT_KEY) # Use the blob client to create the containers in Azure Storage if they # don't yet exist. blob_client.create_container(config.CONTAINER_NAME, fail_on_exist=False) CONTAINER_SAS_URL = build_output_sas_url(config, blob_client) # The collection of data files that are to be processed by the tasks. input_file_path = os.path.join(sys.path[0], _LOCAL_INPUT_FILE) # Upload the data files. input_file = upload_file_to_container(blob_client, config.CONTAINER_NAME, input_file_path, config.STORAGE_ACCESS_DURATION_HRS) # Create a Batch service client. We'll now be interacting with the Batch # service in addition to Storage credentials = batch_auth.SharedKeyCredentials(config.BATCH_ACCOUNT_NAME, config.BATCH_ACCOUNT_KEY) batch_client = batch.BatchServiceClient(credentials, batch_url=config.BATCH_ACCOUNT_URL) try: # Create the pool that will contain the compute nodes that will execute the # tasks. try: create_pool(config, batch_client) print("Created pool: ", config.POOL_ID) except models.BatchErrorException: print("Using pool: ", config.POOL_ID) # Create the job that will run the tasks. create_job(batch_client, config.JOB_ID, config.POOL_ID) # Add the tasks to the job. add_tasks( config, blob_client, batch_client, CONTAINER_SAS_URL, config.JOB_ID, input_file, n_folds, ) if not wait: return # Pause execution until tasks reach Completed state. wait_for_tasks_to_complete( batch_client, config.JOB_ID, datetime.timedelta(hours=config.STORAGE_ACCESS_DURATION_HRS)) _download_files(config, blob_client, config.BATCH_DIRECTORY, n_folds) except models.BatchErrorException as err: print_batch_exception(err) raise err # Clean up storage resources # TODO: re-enable this and delete the output container too # -- print("Deleting container [{}]...".format(input_container_name)) # -- blob_client.delete_container(input_container_name) # Print out some timing info end_time = datetime.datetime.now().replace(microsecond=0) print() print("Sample end: {}".format(end_time)) print("Elapsed time: {}".format(end_time - start_time)) print() # Clean up Batch resources (if the user so chooses). if config.DELETE_POOL_WHEN_DONE: batch_client.pool.delete(config.POOL_ID) if config.DELETE_JOB_WHEN_DONE: batch_client.job.delete(config.JOB_ID)