def add_task_to_start_server(self): constraint = batchmodels.TaskConstraints( retention_time=datetime.timedelta(hours=24), ) user_identity = batch.models.UserIdentity( # user_name='azureuser', auto_user=batch.models.AutoUserSpecification( scope=batch.models.AutoUserScope.pool, elevation_level=batch.models.ElevationLevel.admin)) task = batchmodels.TaskAddParameter( id=helpers.generate_unique_resource_name( f"Server-{str(self.globalTaskCounter)}"), # id=f"Server-{str(self.globalTaskCounter)}", command_line=helpers.wrap_commands_in_shell( 'linux', self.get_start_task_commands()), constraints=constraint, user_identity=user_identity) try: self.globalTaskCounter += 1 self.client.task.add(job_id=self.job_id, task=task) except BatchErrorException as e: self.globalTaskCounter += 1 self.start_mc_server_job_pool(1) self.client.task.add(job_id=self.job_id, task=task)
def create_job(self, job_id, pool_id, total_nodes, is_linux_pool): client = self._get_batch_client() try: pool_info = batchmodels.PoolInformation(pool_id=pool_id) job = batchmodels.JobAddParameter(id=job_id, pool_info=pool_info) try: client.job.add(job) except batchmodels.BatchErrorException as be: if be.error and be.error.code == 'JobExists': pass else: print('Error creating job, code={}, message={}'.format( be.error.code, be.error.message)) raise if is_linux_pool: cmd_line = '/bin/bash -c azure-batch-ses.sh' script = 'azure-batch-ses.sh' script_url = 'https://raw.githubusercontent.com/Azure/azure-deadline/master/CloudProviderPlugin/Scripts/azure-batch-ses.sh' else: cmd_line = 'powershell.exe -file azure-batch-ses.ps1' script = 'azure-batch-ses.ps1' script_url = 'https://raw.githubusercontent.com/Azure/azure-deadline/master/CloudProviderPlugin/Scripts/azure-batch-ses.ps1' task = batchmodels.TaskAddParameter( id='', command_line=cmd_line, resource_files=[batchmodels.ResourceFile(script_url, script)], constraints=batchmodels.TaskConstraints( max_task_retry_count=3), user_identity=batchmodels.UserIdentity( auto_user=batchmodels.AutoUserSpecification( scope=batchmodels.AutoUserScope.pool, elevation_level=batchmodels.ElevationLevel.admin))) for i in range(total_nodes): task.id = str(uuid.uuid4()) client.task.add(job_id=job.id, task=task) except batchmodels.BatchErrorException as be: if be.error: print('Error creating job, code={}, message={}'.format( be.error.code, be.error.message)) if be.error.values: for e in be.error.values: print('Key={}, Value={}'.format(e.key, e.value)) raise
def test_batch_tasks(self, batch_job, **kwargs): client = self.create_sharedkey_client(**kwargs) # Test Create Task with Auto Complete exit_conditions = models.ExitConditions( exit_codes=[models.ExitCodeMapping(1, models.ExitOptions(models.JobAction.terminate))], exit_code_ranges=[models.ExitCodeRangeMapping(2, 4, models.ExitOptions(models.JobAction.disable))], default=models.ExitOptions(models.JobAction.none)) task_param = models.TaskAddParameter( id=self.get_resource_name('batch_task1_'), command_line='cmd /c "echo hello world"', exit_conditions=exit_conditions ) try: client.task.add(batch_job.id, task_param) except models.BatchErrorException as e: message = "{}: ".format(e.error.code, e.error.message) for v in e.error.values: message += "\n{}: {}".format(v.key, v.value) raise Exception(message) task = client.task.get(batch_job.id, task_param.id) self.assertIsInstance(task, models.CloudTask) self.assertEqual(task.exit_conditions.default.job_action, models.JobAction.none) self.assertEqual(task.exit_conditions.exit_codes[0].code, 1) self.assertEqual(task.exit_conditions.exit_codes[0].exit_options.job_action, models.JobAction.terminate) # Test Create Task with Output Files container_url = "https://test.blob.core.windows.net:443/test-container" outputs = [ models.OutputFile( file_pattern="../stdout.txt", destination=models.OutputFileDestination( container=models.OutputFileBlobContainerDestination( container_url=container_url, path="taskLogs/output.txt")), upload_options=models.OutputFileUploadOptions( upload_condition=models.OutputFileUploadCondition.task_completion)), models.OutputFile( file_pattern="../stderr.txt", destination=models.OutputFileDestination( container=models.OutputFileBlobContainerDestination( container_url=container_url, path="taskLogs/error.txt")), upload_options=models.OutputFileUploadOptions( upload_condition=models.OutputFileUploadCondition.task_failure)), ] task_param = models.TaskAddParameter( id=self.get_resource_name('batch_task2_'), command_line='cmd /c "echo hello world"', output_files=outputs ) client.task.add(batch_job.id, task_param) task = client.task.get(batch_job.id, task_param.id) self.assertIsInstance(task, models.CloudTask) self.assertEqual(len(task.output_files), 2) # Test Create Task with Auto User auto_user = models.AutoUserSpecification( scope=models.AutoUserScope.task, elevation_level=models.ElevationLevel.admin) task_param = models.TaskAddParameter( id=self.get_resource_name('batch_task3_'), command_line='cmd /c "echo hello world"', user_identity=models.UserIdentity(auto_user=auto_user) ) client.task.add(batch_job.id, task_param) task = client.task.get(batch_job.id, task_param.id) self.assertIsInstance(task, models.CloudTask) self.assertEqual(task.user_identity.auto_user.scope, models.AutoUserScope.task) self.assertEqual(task.user_identity.auto_user.elevation_level, models.ElevationLevel.admin) # Test Create Task with Token Settings task_param = models.TaskAddParameter( id=self.get_resource_name('batch_task4_'), command_line='cmd /c "echo hello world"', authentication_token_settings=models.AuthenticationTokenSettings( access=[models.AccessScope.job]) ) client.task.add(batch_job.id, task_param) task = client.task.get(batch_job.id, task_param.id) self.assertIsInstance(task, models.CloudTask) self.assertEqual(task.authentication_token_settings.access[0], models.AccessScope.job) # Test Create Task with Container Settings task_param = models.TaskAddParameter( id=self.get_resource_name('batch_task5_'), command_line='cmd /c "echo hello world"', container_settings=models.TaskContainerSettings( image_name='windows_container:latest', registry=models.ContainerRegistry('username', 'password')) ) client.task.add(batch_job.id, task_param) task = client.task.get(batch_job.id, task_param.id) self.assertIsInstance(task, models.CloudTask) self.assertEqual(task.container_settings.image_name, 'windows_container:latest') self.assertEqual(task.container_settings.registry.user_name, 'username') # Test Create Task with Run-As-User task_param = models.TaskAddParameter( id=self.get_resource_name('batch_task6_'), command_line='cmd /c "echo hello world"', user_identity=models.UserIdentity(user_name='task-user') ) client.task.add(batch_job.id, task_param) task = client.task.get(batch_job.id, task_param.id) self.assertIsInstance(task, models.CloudTask) self.assertEqual(task.user_identity.user_name, 'task-user') # Test Add Task Collection tasks = [] for i in range(7, 10): tasks.append(models.TaskAddParameter( self.get_resource_name('batch_task{}_'.format(i)), 'cmd /c "echo hello world"')) result = client.task.add_collection(batch_job.id, tasks) self.assertIsInstance(result, models.TaskAddCollectionResult) self.assertEqual(len(result.value), 3) self.assertEqual(result.value[0].status, models.TaskAddStatus.success) # Test List Tasks tasks = list(client.task.list(batch_job.id)) self.assertEqual(len(tasks), 9) # Test Count Tasks task_counts = client.job.get_task_counts(batch_job.id) self.assertIsInstance(task_counts, models.TaskCounts) self.assertEqual(task_counts.completed, 0) self.assertEqual(task_counts.succeeded, 0) self.assertEqual(task_counts.validation_status, models.TaskCountValidationStatus.validated) # Test Terminate Task response = client.task.terminate(batch_job.id, task_param.id) self.assertIsNone(response) task = client.task.get(batch_job.id, task_param.id) self.assertEqual(task.state, models.TaskState.completed) # Test Reactivate Task response = client.task.reactivate(batch_job.id, task_param.id) self.assertIsNone(response) task = client.task.get(batch_job.id, task_param.id) self.assertEqual(task.state, models.TaskState.active) # Test Update Task response = client.task.update( batch_job.id, task_param.id, constraints=models.TaskConstraints(max_task_retry_count=1)) self.assertIsNone(response) # Test Get Subtasks # TODO: Test with actual subtasks subtasks = client.task.list_subtasks(batch_job.id, task_param.id) self.assertIsInstance(subtasks, models.CloudTaskListSubtasksResult) self.assertEqual(subtasks.value, []) # Test Delete Task response = client.task.delete(batch_job.id, task_param.id) self.assertIsNone(response)
def create_merge_task(frame, task_id, job_id, depend_start, depend_end): """ Azure Batch task that executes the ImageMagick `convert` command line to combine all of the output tiles into a single output image. This task uses the task dependency model to make sure it doesn't execute before it's dependent tasks have completed. This way we know all of the output image tiles will exist. :param frame: Frame number of the scene that this merge task is processing. :type frame: int :param task_id: Identifier of the task. :type task_id: str :param job_id: Unique identifier of the job. Job identifiers are unique within a single Azure Batch account. :type job_id: str :param depend_start: First task id of the dependency sequence. If each frame is split into 16 tiles, then every 17th task will be a merge task and that merge task will be dependent on the preceeding 16 tasks. tile tasks 1 - 16, then merge, then tiles 18 - 34, then merge, etc. :type depend_start: int :param depend_end: Final task id of the dependency sequence. Explanation for param `depend_start` applies here as well. :type depend_end: int """ x_tiles = int(os.environ["X_TILES"]) y_tiles = int(os.environ["X_TILES"]) output_sas = os.environ["OUTPUT_CONTAINER_SAS"] working_dir = os.environ["AZ_BATCH_TASK_WORKING_DIR"] output_format = os.environ["OUTPUT_FORMAT"] print("working_dir: {}".format(working_dir)) # crop to border means we need to use montage to tile the images. false means # we can use convert -flatten to layer the images with transparent backgrounds # convert is faster but needs RGBA crop = os.environ["CROP_TO_BORDER"].lower() if crop == "true": command_line = montage_command(frame, x_tiles, y_tiles, output_format) else: command_line = convert_command(frame, output_format) print("merge task command line: {}".format(command_line)) return models.TaskAddParameter( id=pad_number(task_id, PAD_LEN_ID), display_name="frame: {} - merge task".format(frame), command_line=os_specific_command_line(command_line), constraints=models.TaskConstraints(max_task_retry_count=2), environment_settings=[ models.EnvironmentSetting("X_TILES", str(x_tiles)), models.EnvironmentSetting("Y_TILES", str(y_tiles)) ], depends_on=models.TaskDependencies( task_ids=get_dependent_tasks(depend_start, depend_end)), resource_files=get_resource_files(x_tiles, y_tiles, frame), output_files=[ models.OutputFile( file_pattern="../stdout.txt", destination=models.OutputFileDestination( container=models.OutputFileBlobContainerDestination( container_url=output_sas, path="{}/logs/frame-{}/merge.stdout.log".format( job_id, pad_number(frame, PAD_LEN_FRAME)))), upload_options=models.OutputFileUploadOptions( models.OutputFileUploadCondition.task_completion)), models.OutputFile( file_pattern="../stderr.txt", destination=models.OutputFileDestination( container=models.OutputFileBlobContainerDestination( container_url=output_sas, path="{}/logs/frame-{}/merge.stderr.log".format( job_id, pad_number(frame, PAD_LEN_FRAME)))), upload_options=models.OutputFileUploadOptions( models.OutputFileUploadCondition.task_completion)), models.OutputFile( file_pattern="frame_*", destination=models.OutputFileDestination( container=models.OutputFileBlobContainerDestination( container_url=output_sas, path="{}/outputs/final".format(job_id))), upload_options=models.OutputFileUploadOptions( models.OutputFileUploadCondition.task_success)) ])
def create_task(frame, task_id, job_id, tile_num, current_x, current_y): """ Azure Batch task that renders the given tile. Run Blender from the command line and pass in the job manager script and the blend file. :param frame: Frame number of the scene that this merge task is processing. :type frame: int :param task_id: Identifier of the task. :type task_id: str :param job_id: Unique identifier of the job. Job identifiers are unique within a single Azure Batch account. :type job_id: str :param tile_num: Number of the current tile. :type tile_num: int :param current_x: X value of the current tile, used to generate the render border. :type current_x: int :param current_y: Y value of the current tile, used to generate the render border. :type current_y: int """ blend_file = os.environ["BLEND_FILE"] output_sas = os.environ["OUTPUT_CONTAINER_SAS"] optionalParams = os.environ["OPTIONAL_PARAMS"] command_line = blender_command(blend_file, optionalParams) # only print this once if task_id == 1: print("tile task command line: {}".format(command_line)) return models.TaskAddParameter( id=pad_number(task_id, PAD_LEN_ID), display_name="frame: {}, tile: {}".format(frame, tile_num), command_line=os_specific_command_line(command_line), constraints=models.TaskConstraints(max_task_retry_count=2), environment_settings=[ models.EnvironmentSetting("X_TILES", os.environ["X_TILES"]), models.EnvironmentSetting("Y_TILES", os.environ["Y_TILES"]), models.EnvironmentSetting("CROP_TO_BORDER", os.environ["CROP_TO_BORDER"]), models.EnvironmentSetting("OUTPUT_FORMAT", os.environ["OUTPUT_FORMAT"]), models.EnvironmentSetting("BLEND_FILE", os.environ["BLEND_FILE"]), models.EnvironmentSetting("CURRENT_FRAME", str(frame)), models.EnvironmentSetting("CURRENT_TILE", str(tile_num)), models.EnvironmentSetting("CURRENT_X", str(current_x)), models.EnvironmentSetting("CURRENT_Y", str(current_y)) ], resource_files=[ models.ResourceFile( "https://raw.githubusercontent.com/Azure/BatchExplorer-data/master/ncj/blender/scripts/python-task-manager.py", "scripts/python-task-manager.py") ], output_files=[ models.OutputFile( file_pattern="../stdout.txt", destination=models.OutputFileDestination( container=models.OutputFileBlobContainerDestination( container_url=output_sas, path="{}/logs/frame-{}/tile-{}.stdout.log".format( job_id, pad_number(frame, PAD_LEN_FRAME), pad_number(tile_num, PAD_LEN_TILE)))), upload_options=models.OutputFileUploadOptions( models.OutputFileUploadCondition.task_completion)), models.OutputFile( file_pattern="../stderr.txt", destination=models.OutputFileDestination( container=models.OutputFileBlobContainerDestination( container_url=output_sas, path="{}/logs/frame-{}/tile-{}.stderr.log".format( job_id, pad_number(frame, PAD_LEN_FRAME), pad_number(tile_num, PAD_LEN_TILE)))), upload_options=models.OutputFileUploadOptions( models.OutputFileUploadCondition.task_completion)), models.OutputFile( file_pattern="../fileuploaderr.txt", destination=models.OutputFileDestination( container=models.OutputFileBlobContainerDestination( container_url=output_sas, path="{}/logs/frame-{}/tile-{}.file_upload_stderr.log". format(job_id, pad_number(frame, PAD_LEN_FRAME), pad_number(tile_num, PAD_LEN_TILE)))), upload_options=models.OutputFileUploadOptions( models.OutputFileUploadCondition.task_completion)), models.OutputFile( file_pattern="../fileuploadout.txt", destination=models.OutputFileDestination( container=models.OutputFileBlobContainerDestination( container_url=output_sas, path="{}/logs/frame-{}/tile-{}.file_upload_stdout.log". format(job_id, pad_number(frame, PAD_LEN_FRAME), pad_number(tile_num, PAD_LEN_TILE)))), upload_options=models.OutputFileUploadOptions( models.OutputFileUploadCondition.task_completion)), models.OutputFile( file_pattern="tile_*", destination=models.OutputFileDestination( container=models.OutputFileBlobContainerDestination( container_url=output_sas, path="{}/outputs/frame-{}".format( job_id, pad_number(frame, PAD_LEN_FRAME)))), upload_options=models.OutputFileUploadOptions( models.OutputFileUploadCondition.task_success)) ])
def add_task(self, job_id: str, default_max_tries=None): """ Adds a task for each input file in the collection to the specified job. :param str job_id: The ID of the job to which to add the tasks. created for each input file. :param int default_max_tries: Fallback max tries. :output task: Azure Batch task """ from azure.batch import models as batchmodels task_id = self.task_definition.get('id', job_id) display_name = self.task_definition.get('displayName', task_id) logging.info('Adding {} tasks to job [{}]...'.format(task_id, job_id)) container_settings = batchmodels.TaskContainerSettings( image_name=self.image, container_run_options='--rm') platform = self.conf[utils.PLATFORM] environment_settings = [ batchmodels.EnvironmentSetting(name='AZURE_SUBSCRIPTION_ID', value=platform['subscription']), batchmodels.EnvironmentSetting(name='AZURE_STORAGE_ACCOUNT', value=platform['storage_account']), batchmodels.EnvironmentSetting( name='AZURE_STORAGE_CONTAINER', value=platform['storage_container']), batchmodels.EnvironmentSetting( name='AZURE_STORAGE_CONNECTION_STRING', value=platform['storage_connection_string']), ] if 'environmentSettings' in self.task_definition and self.task_definition[ 'environmentSettings'] is not None: environment_settings.extend([ batchmodels.EnvironmentSetting(**setting) for setting in self.task_definition['environmentSettings'] ]) constraints = None if 'constraints' in self.task_definition and self.task_definition[ 'constraints']: constraints = batchmodels.TaskConstraints( max_wall_clock_time=self.task_definition['constraints'].get( 'maxWallClockTime', "P1D"), max_task_retry_count=self.task_definition['constraints'].get( 'maxTaskRetryCount', default_max_tries), retention_time=self.task_definition['constraints'].get( 'retentionTime', "P1D"), ), user_identity = batchmodels.UserIdentity( auto_user=batchmodels.AutoUserSpecification( scope=batchmodels.AutoUserScope.pool, elevation_level=batchmodels.ElevationLevel.admin)) task = batchmodels.TaskAddParameter( id=task_id, display_name=display_name, command_line=self.task_definition['commandLine'], constraints=constraints[0], container_settings=container_settings, environment_settings=environment_settings, user_identity=user_identity, ) for validation in task.validate(): logging.info(validation) self.batch_client.task.add(job_id=job_id, task=task) return task
def generate_task(spark_client, container_id, application, remote=False): resource_files = [] # The application provided is not hosted remotely and therefore must be uploaded if not remote: app_resource_file = helpers.upload_file_to_container( container_name=container_id, application_name=application.name, file_path=application.application, blob_client=spark_client.blob_client, use_full_path=False, ) # Upload application file resource_files.append(app_resource_file) application.application = "$AZ_BATCH_TASK_WORKING_DIR/" + os.path.basename( application.application) # Upload dependent JARS jar_resource_file_paths = [] for jar in application.jars: current_jar_resource_file_path = helpers.upload_file_to_container( container_name=container_id, application_name=application.name, file_path=jar, blob_client=spark_client.blob_client, use_full_path=False, ) jar_resource_file_paths.append(current_jar_resource_file_path) resource_files.append(current_jar_resource_file_path) # Upload dependent python files py_files_resource_file_paths = [] for py_file in application.py_files: current_py_files_resource_file_path = helpers.upload_file_to_container( container_name=container_id, application_name=application.name, file_path=py_file, blob_client=spark_client.blob_client, use_full_path=False, ) py_files_resource_file_paths.append( current_py_files_resource_file_path) resource_files.append(current_py_files_resource_file_path) # Upload other dependent files files_resource_file_paths = [] for file in application.files: files_resource_file_path = helpers.upload_file_to_container( container_name=container_id, application_name=application.name, file_path=file, blob_client=spark_client.blob_client, use_full_path=False, ) files_resource_file_paths.append(files_resource_file_path) resource_files.append(files_resource_file_path) # Upload application definition application.jars = [os.path.basename(jar) for jar in application.jars] application.py_files = [ os.path.basename(py_files) for py_files in application.py_files ] application.files = [ os.path.basename(files) for files in application.files ] application_definition_file = helpers.upload_text_to_container( container_name=container_id, application_name=application.name, file_path="application.yaml", content=yaml.dump(vars(application)), blob_client=spark_client.blob_client, ) resource_files.append(application_definition_file) # create command to submit task task_cmd = CommandBuilder("sudo docker exec") task_cmd.add_argument("-i") task_cmd.add_option( "-e", "AZ_BATCH_TASK_WORKING_DIR=$AZ_BATCH_TASK_WORKING_DIR") task_cmd.add_option("-e", "STORAGE_LOGS_CONTAINER={0}".format(container_id)) task_cmd.add_argument("spark /bin/bash >> output.log 2>&1") task_cmd.add_argument( r'-c "source ~/.bashrc; ' r"export PYTHONPATH=$PYTHONPATH:\$AZTK_WORKING_DIR; " r"cd \$AZ_BATCH_TASK_WORKING_DIR; " r'\$AZTK_WORKING_DIR/.aztk-env/.venv/bin/python \$AZTK_WORKING_DIR/aztk/node_scripts/submit.py"' ) # Create task task = batch_models.TaskAddParameter( id=application.name, command_line=helpers.wrap_commands_in_shell([task_cmd.to_str()]), resource_files=resource_files, constraints=batch_models.TaskConstraints( max_task_retry_count=application.max_retry_count), user_identity=batch_models.UserIdentity( auto_user=batch_models.AutoUserSpecification( scope=batch_models.AutoUserScope.task, elevation_level=batch_models.ElevationLevel.admin)), ) return task
def generate_task(spark_client, container_id, application): resource_files = [] app_resource_file = helpers.upload_file_to_container( container_name=container_id, application_name=application.name, file_path=application.application, blob_client=spark_client.blob_client, use_full_path=False) # Upload application file resource_files.append(app_resource_file) # Upload dependent JARS jar_resource_file_paths = [] for jar in application.jars: current_jar_resource_file_path = helpers.upload_file_to_container( container_name=container_id, application_name=application.name, file_path=jar, blob_client=spark_client.blob_client, use_full_path=False) jar_resource_file_paths.append(current_jar_resource_file_path) resource_files.append(current_jar_resource_file_path) # Upload dependent python files py_files_resource_file_paths = [] for py_file in application.py_files: current_py_files_resource_file_path = helpers.upload_file_to_container( container_name=container_id, application_name=application.name, file_path=py_file, blob_client=spark_client.blob_client, use_full_path=False) py_files_resource_file_paths.append( current_py_files_resource_file_path) resource_files.append(current_py_files_resource_file_path) # Upload other dependent files files_resource_file_paths = [] for file in application.files: files_resource_file_path = helpers.upload_file_to_container( container_name=container_id, application_name=application.name, file_path=file, blob_client=spark_client.blob_client, use_full_path=False) files_resource_file_paths.append(files_resource_file_path) resource_files.append(files_resource_file_path) # Upload application definition application.application = os.path.basename(application.application) application.jars = [os.path.basename(jar) for jar in application.jars] application.py_files = [ os.path.basename(py_files) for py_files in application.py_files ] application.files = [ os.path.basename(files) for files in application.files ] application_definition_file = helpers.upload_text_to_container( container_name=container_id, application_name=application.name, file_path='application.yaml', content=yaml.dump(vars(application)), blob_client=spark_client.blob_client) resource_files.append(application_definition_file) # create command to submit task task_cmd = CommandBuilder('sudo docker exec') task_cmd.add_argument('-i') task_cmd.add_option( '-e', 'AZ_BATCH_TASK_WORKING_DIR=$AZ_BATCH_TASK_WORKING_DIR') task_cmd.add_option('-e', 'STORAGE_LOGS_CONTAINER={0}'.format(container_id)) task_cmd.add_argument('spark /bin/bash >> output.log 2>&1') task_cmd.add_argument('-c "source ~/.bashrc; '\ 'cd $AZ_BATCH_TASK_WORKING_DIR; ' \ '\$(pyenv root)/versions/\$AZTK_PYTHON_VERSION/bin/python ' \ '\$DOCKER_WORKING_DIR/aztk/node_scripts/submit.py"') # Create task task = batch_models.TaskAddParameter( id=application.name, command_line=helpers.wrap_commands_in_shell([task_cmd.to_str()]), resource_files=resource_files, constraints=batch_models.TaskConstraints( max_task_retry_count=application.max_retry_count), user_identity=batch_models.UserIdentity( auto_user=batch_models.AutoUserSpecification( scope=batch_models.AutoUserScope.task, elevation_level=batch_models.ElevationLevel.admin))) return task
def add_task(batch_service_client, job_id, input_files, output_container_sas_url, task_name, input_file_dir): """ Adds a task for each input file in the collection to the specified job. :param batch_service_client: A Batch service client. :type batch_service_client: `azure.batch.BatchServiceClient` :param str job_id: The ID of the job to which to add the tasks. :param list input_files: A collection of input files. One task will be created for each input file. :param output_container_sas_token: A SAS token granting write access to the specified Azure Blob storage container. """ tasks = list() # my own tasks section logging.info('Adding dashcam task [{task}] to job [{job}]...'.format( task=task_name, job=job_id)) output_file_path = "/mnt/batch/tasks/workitems/{jobname}/job-1/{taskname}/wd".format( jobname=job_id, taskname=task_name) logging.info('input_file_dir: {}'.format(input_file_dir)) logging.info('output_file_path: {}'.format(output_file_path)) # Command uses the script from https://github.com/ehendrix23/tesla_dashcam command = '''tesla_dashcam --no-timestamp --monitor_trigger {input} --output {output} --motion_only --mirror --layout DIAMOND --fps 33 --quality HIGH --slowdown 5'''.format( input=input_file_dir, output=output_file_path) logging.info('command: {}'.format(command)) tasks.append( batch.models.TaskAddParameter( id=task_name, command_line=command, constraints=batchmodels.TaskConstraints( max_wall_clock_time=datetime.timedelta(minutes=60), max_task_retry_count=1), resource_files=input_files, output_files=[ batchmodels.OutputFile( file_pattern='*.mp4', destination=batchmodels.OutputFileDestination( container=batchmodels. OutputFileBlobContainerDestination( path='', container_url= output_container_sas_url #os.environ["_SAS"] )), upload_options=batchmodels.OutputFileUploadOptions( upload_condition=batchmodels.OutputFileUploadCondition. task_success)), batchmodels.OutputFile( file_pattern='../std*.txt', destination=batchmodels.OutputFileDestination( container=batchmodels. OutputFileBlobContainerDestination( path=task_name, container_url= output_container_sas_url #os.environ["_SAS"] )), upload_options=batchmodels.OutputFileUploadOptions( upload_condition=batchmodels.OutputFileUploadCondition. task_success)), batchmodels.OutputFile( file_pattern='../fileupload*.txt', destination=batchmodels.OutputFileDestination( container=batchmodels. OutputFileBlobContainerDestination( path=task_name, container_url= output_container_sas_url #os.environ["_SAS"] )), upload_options=batchmodels.OutputFileUploadOptions( upload_condition=batchmodels.OutputFileUploadCondition. task_success)) ])) batch_service_client.task.add_collection(job_id, tasks)
def submit_application(spark_client, cluster_id, application, wait: bool = False): """ Submit a spark app """ resource_files = [] app_resource_file = helpers.upload_file_to_container(container_name=application.name, file_path=application.application, blob_client=spark_client.blob_client, use_full_path=False) # Upload application file resource_files.append(app_resource_file) # Upload dependent JARS jar_resource_file_paths = [] for jar in application.jars: current_jar_resource_file_path = helpers.upload_file_to_container(container_name=application.name, file_path=jar, blob_client=spark_client.blob_client, use_full_path=False) jar_resource_file_paths.append(current_jar_resource_file_path) resource_files.append(current_jar_resource_file_path) # Upload dependent python files py_files_resource_file_paths = [] for py_file in application.py_files: current_py_files_resource_file_path = helpers.upload_file_to_container(container_name=application.name, file_path=py_file, blob_client=spark_client.blob_client, use_full_path=False) py_files_resource_file_paths.append( current_py_files_resource_file_path) resource_files.append(current_py_files_resource_file_path) # Upload other dependent files files_resource_file_paths = [] for file in application.files: files_resource_file_path = helpers.upload_file_to_container(container_name=application.name, file_path=file, blob_client=spark_client.blob_client, use_full_path=False) files_resource_file_paths.append(files_resource_file_path) resource_files.append(files_resource_file_path) # create command to submit task cmd = __app_submit_cmd( spark_client=spark_client, cluster_id=cluster_id, name=application.name, app=app_resource_file.file_path, app_args=application.application_args, main_class=application.main_class, jars=[jar_resource_file_path.file_path for jar_resource_file_path in jar_resource_file_paths], py_files=[py_files_resource.file_path for py_files_resource in py_files_resource_file_paths], files=[file_resource_file_path.file_path for file_resource_file_path in files_resource_file_paths], driver_java_options=application.driver_java_options, driver_library_path=application.driver_library_path, driver_class_path=application.driver_class_path, driver_memory=application.driver_memory, executor_memory=application.executor_memory, driver_cores=application.driver_cores, executor_cores=application.executor_cores) # Get cluster size cluster = spark_client.get_cluster(cluster_id) # Affinitize task to master node # master_node_affinity_id = helpers.get_master_node_id(cluster_id, spark_client.batch_client) rls = spark_client.get_remote_login_settings(cluster.id, cluster.master_node_id) # Create task task = batch_models.TaskAddParameter( id=application.name, affinity_info=batch_models.AffinityInformation( affinity_id=cluster.master_node_id), command_line=helpers.wrap_commands_in_shell(cmd), resource_files=resource_files, constraints=batch_models.TaskConstraints( max_task_retry_count=application.max_retry_count), user_identity=batch_models.UserIdentity( auto_user=batch_models.AutoUserSpecification( scope=batch_models.AutoUserScope.task, elevation_level=batch_models.ElevationLevel.admin)) ) # Add task to batch job (which has the same name as cluster_id) job_id = cluster_id spark_client.batch_client.task.add(job_id=job_id, task=task) if wait: helpers.wait_for_task_to_complete(job_id=job_id, task_id=task.id, batch_client=spark_client.batch_client)
def create_task(dataset, command, dependencies, max_wall_clock, production): if production: container = RESULTS_CONTAINER else: container = TEST_RESULTS_CONTAINER + "/" + \ generate_task_name(dataset.name) output_files = [ # Upload results batch_models.OutputFile( file_pattern="$AZ_BATCH_TASK_WORKING_DIR/results/**/*", upload_options=batch_models.OutputFileUploadOptions( upload_condition=batch_models.OutputFileUploadCondition. task_success), destination=batch_models.OutputFileDestination( container=batch_models.OutputFileBlobContainerDestination( path=dataset.data_dir, container_url=container + SAS_TOKEN))), batch_models.OutputFile( file_pattern= f"$AZ_BATCH_NODE_ROOT_DIR/fsmounts/{FILE_SHARE_NAME}/*.csv", upload_options=batch_models.OutputFileUploadOptions( upload_condition=batch_models.OutputFileUploadCondition. task_success), destination=batch_models.OutputFileDestination( container=batch_models.OutputFileBlobContainerDestination( container_url=container + SAS_TOKEN))), batch_models.OutputFile( file_pattern= f"$AZ_BATCH_NODE_ROOT_DIR/fsmounts/{FILE_SHARE_NAME}/last-update/*", upload_options=batch_models.OutputFileUploadOptions( upload_condition=batch_models.OutputFileUploadCondition. task_success), destination=batch_models.OutputFileDestination( container=batch_models.OutputFileBlobContainerDestination( path="last-update", container_url=container + SAS_TOKEN))), # Upload stderr and stdout batch_models.OutputFile( file_pattern="$AZ_BATCH_TASK_DIR/std*.txt", upload_options=batch_models.OutputFileUploadOptions( upload_condition=batch_models.OutputFileUploadCondition. task_completion), destination=batch_models.OutputFileDestination( container=batch_models.OutputFileBlobContainerDestination( path=DATETIME_NOWISH + "/" + generate_task_name(dataset.name), container_url=PROCESS_LOG_CONTAINER + "/" + SAS_TOKEN))) ] return batch_models.TaskAddParameter( id=generate_task_name(dataset.name), display_name=(dataset.name + "_python_script_job"), command_line=command, resource_files=[ batch_models.ResourceFile(storage_container_url=CONFIG_CONTAINER + SAS_TOKEN, blob_prefix=dataset.name + CONFIG_FILE) ], depends_on=batch_models.TaskDependencies(task_ids=dependencies), user_identity=batch_models.UserIdentity( auto_user=batch_models.AutoUserSpecification( scope='pool', elevation_level='admin')), container_settings=batch_models.TaskContainerSettings( image_name=DOCKER_CONTAINER_URL, container_run_options='-w /home/rstudio/covid-rt-estimates'), constraints=batch_models.TaskConstraints( max_wall_clock_time=datetime.timedelta(minutes=max_wall_clock)), output_files=output_files)