def add_task_to_start_server(self):
constraint = batchmodels.TaskConstraints(
retention_time=datetime.timedelta(hours=24), )
user_identity = batch.models.UserIdentity(
# user_name='azureuser',
auto_user=batch.models.AutoUserSpecification(
scope=batch.models.AutoUserScope.pool,
elevation_level=batch.models.ElevationLevel.admin))
task = batchmodels.TaskAddParameter(
id=helpers.generate_unique_resource_name(
f"Server-{str(self.globalTaskCounter)}"),
# id=f"Server-{str(self.globalTaskCounter)}",
command_line=helpers.wrap_commands_in_shell(
'linux', self.get_start_task_commands()),
constraints=constraint,
user_identity=user_identity)
try:
self.globalTaskCounter += 1
self.client.task.add(job_id=self.job_id, task=task)
except BatchErrorException as e:
self.globalTaskCounter += 1
self.start_mc_server_job_pool(1)
self.client.task.add(job_id=self.job_id, task=task)
def create_job(self, job_id, pool_id, total_nodes, is_linux_pool):
client = self._get_batch_client()
try:
pool_info = batchmodels.PoolInformation(pool_id=pool_id)
job = batchmodels.JobAddParameter(id=job_id, pool_info=pool_info)
try:
client.job.add(job)
except batchmodels.BatchErrorException as be:
if be.error and be.error.code == 'JobExists':
pass
else:
print('Error creating job, code={}, message={}'.format(
be.error.code, be.error.message))
raise
if is_linux_pool:
cmd_line = '/bin/bash -c azure-batch-ses.sh'
script = 'azure-batch-ses.sh'
script_url = 'https://raw.githubusercontent.com/Azure/azure-deadline/master/CloudProviderPlugin/Scripts/azure-batch-ses.sh'
else:
cmd_line = 'powershell.exe -file azure-batch-ses.ps1'
script = 'azure-batch-ses.ps1'
script_url = 'https://raw.githubusercontent.com/Azure/azure-deadline/master/CloudProviderPlugin/Scripts/azure-batch-ses.ps1'
task = batchmodels.TaskAddParameter(
id='',
command_line=cmd_line,
resource_files=[batchmodels.ResourceFile(script_url, script)],
constraints=batchmodels.TaskConstraints(
max_task_retry_count=3),
user_identity=batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin)))
for i in range(total_nodes):
task.id = str(uuid.uuid4())
client.task.add(job_id=job.id, task=task)
except batchmodels.BatchErrorException as be:
if be.error:
print('Error creating job, code={}, message={}'.format(
be.error.code, be.error.message))
if be.error.values:
for e in be.error.values:
print('Key={}, Value={}'.format(e.key, e.value))
raise
def test_batch_tasks(self, batch_job, **kwargs):
client = self.create_sharedkey_client(**kwargs)
# Test Create Task with Auto Complete
exit_conditions = models.ExitConditions(
exit_codes=[models.ExitCodeMapping(1, models.ExitOptions(models.JobAction.terminate))],
exit_code_ranges=[models.ExitCodeRangeMapping(2, 4, models.ExitOptions(models.JobAction.disable))],
default=models.ExitOptions(models.JobAction.none))
task_param = models.TaskAddParameter(
id=self.get_resource_name('batch_task1_'),
command_line='cmd /c "echo hello world"',
exit_conditions=exit_conditions
)
try:
client.task.add(batch_job.id, task_param)
except models.BatchErrorException as e:
message = "{}: ".format(e.error.code, e.error.message)
for v in e.error.values:
message += "\n{}: {}".format(v.key, v.value)
raise Exception(message)
task = client.task.get(batch_job.id, task_param.id)
self.assertIsInstance(task, models.CloudTask)
self.assertEqual(task.exit_conditions.default.job_action, models.JobAction.none)
self.assertEqual(task.exit_conditions.exit_codes[0].code, 1)
self.assertEqual(task.exit_conditions.exit_codes[0].exit_options.job_action, models.JobAction.terminate)
# Test Create Task with Output Files
container_url = "https://test.blob.core.windows.net:443/test-container"
outputs = [
models.OutputFile(
file_pattern="../stdout.txt",
destination=models.OutputFileDestination(
container=models.OutputFileBlobContainerDestination(
container_url=container_url, path="taskLogs/output.txt")),
upload_options=models.OutputFileUploadOptions(
upload_condition=models.OutputFileUploadCondition.task_completion)),
models.OutputFile(
file_pattern="../stderr.txt",
destination=models.OutputFileDestination(
container=models.OutputFileBlobContainerDestination(
container_url=container_url, path="taskLogs/error.txt")),
upload_options=models.OutputFileUploadOptions(
upload_condition=models.OutputFileUploadCondition.task_failure)),
]
task_param = models.TaskAddParameter(
id=self.get_resource_name('batch_task2_'),
command_line='cmd /c "echo hello world"',
output_files=outputs
)
client.task.add(batch_job.id, task_param)
task = client.task.get(batch_job.id, task_param.id)
self.assertIsInstance(task, models.CloudTask)
self.assertEqual(len(task.output_files), 2)
# Test Create Task with Auto User
auto_user = models.AutoUserSpecification(
scope=models.AutoUserScope.task,
elevation_level=models.ElevationLevel.admin)
task_param = models.TaskAddParameter(
id=self.get_resource_name('batch_task3_'),
command_line='cmd /c "echo hello world"',
user_identity=models.UserIdentity(auto_user=auto_user)
)
client.task.add(batch_job.id, task_param)
task = client.task.get(batch_job.id, task_param.id)
self.assertIsInstance(task, models.CloudTask)
self.assertEqual(task.user_identity.auto_user.scope, models.AutoUserScope.task)
self.assertEqual(task.user_identity.auto_user.elevation_level, models.ElevationLevel.admin)
# Test Create Task with Token Settings
task_param = models.TaskAddParameter(
id=self.get_resource_name('batch_task4_'),
command_line='cmd /c "echo hello world"',
authentication_token_settings=models.AuthenticationTokenSettings(
access=[models.AccessScope.job])
)
client.task.add(batch_job.id, task_param)
task = client.task.get(batch_job.id, task_param.id)
self.assertIsInstance(task, models.CloudTask)
self.assertEqual(task.authentication_token_settings.access[0], models.AccessScope.job)
# Test Create Task with Container Settings
task_param = models.TaskAddParameter(
id=self.get_resource_name('batch_task5_'),
command_line='cmd /c "echo hello world"',
container_settings=models.TaskContainerSettings(
image_name='windows_container:latest',
registry=models.ContainerRegistry('username', 'password'))
)
client.task.add(batch_job.id, task_param)
task = client.task.get(batch_job.id, task_param.id)
self.assertIsInstance(task, models.CloudTask)
self.assertEqual(task.container_settings.image_name, 'windows_container:latest')
self.assertEqual(task.container_settings.registry.user_name, 'username')
# Test Create Task with Run-As-User
task_param = models.TaskAddParameter(
id=self.get_resource_name('batch_task6_'),
command_line='cmd /c "echo hello world"',
user_identity=models.UserIdentity(user_name='task-user')
)
client.task.add(batch_job.id, task_param)
task = client.task.get(batch_job.id, task_param.id)
self.assertIsInstance(task, models.CloudTask)
self.assertEqual(task.user_identity.user_name, 'task-user')
# Test Add Task Collection
tasks = []
for i in range(7, 10):
tasks.append(models.TaskAddParameter(
self.get_resource_name('batch_task{}_'.format(i)), 'cmd /c "echo hello world"'))
result = client.task.add_collection(batch_job.id, tasks)
self.assertIsInstance(result, models.TaskAddCollectionResult)
self.assertEqual(len(result.value), 3)
self.assertEqual(result.value[0].status, models.TaskAddStatus.success)
# Test List Tasks
tasks = list(client.task.list(batch_job.id))
self.assertEqual(len(tasks), 9)
# Test Count Tasks
task_counts = client.job.get_task_counts(batch_job.id)
self.assertIsInstance(task_counts, models.TaskCounts)
self.assertEqual(task_counts.completed, 0)
self.assertEqual(task_counts.succeeded, 0)
self.assertEqual(task_counts.validation_status, models.TaskCountValidationStatus.validated)
# Test Terminate Task
response = client.task.terminate(batch_job.id, task_param.id)
self.assertIsNone(response)
task = client.task.get(batch_job.id, task_param.id)
self.assertEqual(task.state, models.TaskState.completed)
# Test Reactivate Task
response = client.task.reactivate(batch_job.id, task_param.id)
self.assertIsNone(response)
task = client.task.get(batch_job.id, task_param.id)
self.assertEqual(task.state, models.TaskState.active)
# Test Update Task
response = client.task.update(
batch_job.id, task_param.id,
constraints=models.TaskConstraints(max_task_retry_count=1))
self.assertIsNone(response)
# Test Get Subtasks
# TODO: Test with actual subtasks
subtasks = client.task.list_subtasks(batch_job.id, task_param.id)
self.assertIsInstance(subtasks, models.CloudTaskListSubtasksResult)
self.assertEqual(subtasks.value, [])
# Test Delete Task
response = client.task.delete(batch_job.id, task_param.id)
self.assertIsNone(response)
def create_merge_task(frame, task_id, job_id, depend_start, depend_end):
"""
Azure Batch task that executes the ImageMagick `convert` command
line to combine all of the output tiles into a single output image.
This task uses the task dependency model to make sure it
doesn't execute before it's dependent tasks have completed. This way
we know all of the output image tiles will exist.
:param frame: Frame number of the scene that this merge task is
processing.
:type frame: int
:param task_id: Identifier of the task.
:type task_id: str
:param job_id: Unique identifier of the job. Job identifiers are unique
within a single Azure Batch account.
:type job_id: str
:param depend_start: First task id of the dependency sequence. If each
frame is split into 16 tiles, then every 17th task will be a merge task
and that merge task will be dependent on the preceeding 16 tasks.
tile tasks 1 - 16, then merge, then tiles 18 - 34, then merge, etc.
:type depend_start: int
:param depend_end: Final task id of the dependency sequence. Explanation
for param `depend_start` applies here as well.
:type depend_end: int
"""
x_tiles = int(os.environ["X_TILES"])
y_tiles = int(os.environ["X_TILES"])
output_sas = os.environ["OUTPUT_CONTAINER_SAS"]
working_dir = os.environ["AZ_BATCH_TASK_WORKING_DIR"]
output_format = os.environ["OUTPUT_FORMAT"]
print("working_dir: {}".format(working_dir))
# crop to border means we need to use montage to tile the images. false means
# we can use convert -flatten to layer the images with transparent backgrounds
# convert is faster but needs RGBA
crop = os.environ["CROP_TO_BORDER"].lower()
if crop == "true":
command_line = montage_command(frame, x_tiles, y_tiles, output_format)
else:
command_line = convert_command(frame, output_format)
print("merge task command line: {}".format(command_line))
return models.TaskAddParameter(
id=pad_number(task_id, PAD_LEN_ID),
display_name="frame: {} - merge task".format(frame),
command_line=os_specific_command_line(command_line),
constraints=models.TaskConstraints(max_task_retry_count=2),
environment_settings=[
models.EnvironmentSetting("X_TILES", str(x_tiles)),
models.EnvironmentSetting("Y_TILES", str(y_tiles))
],
depends_on=models.TaskDependencies(
task_ids=get_dependent_tasks(depend_start, depend_end)),
resource_files=get_resource_files(x_tiles, y_tiles, frame),
output_files=[
models.OutputFile(
file_pattern="../stdout.txt",
destination=models.OutputFileDestination(
container=models.OutputFileBlobContainerDestination(
container_url=output_sas,
path="{}/logs/frame-{}/merge.stdout.log".format(
job_id, pad_number(frame, PAD_LEN_FRAME)))),
upload_options=models.OutputFileUploadOptions(
models.OutputFileUploadCondition.task_completion)),
models.OutputFile(
file_pattern="../stderr.txt",
destination=models.OutputFileDestination(
container=models.OutputFileBlobContainerDestination(
container_url=output_sas,
path="{}/logs/frame-{}/merge.stderr.log".format(
job_id, pad_number(frame, PAD_LEN_FRAME)))),
upload_options=models.OutputFileUploadOptions(
models.OutputFileUploadCondition.task_completion)),
models.OutputFile(
file_pattern="frame_*",
destination=models.OutputFileDestination(
container=models.OutputFileBlobContainerDestination(
container_url=output_sas,
path="{}/outputs/final".format(job_id))),
upload_options=models.OutputFileUploadOptions(
models.OutputFileUploadCondition.task_success))
])
def create_task(frame, task_id, job_id, tile_num, current_x, current_y):
"""
Azure Batch task that renders the given tile. Run Blender from the command
line and pass in the job manager script and the blend file.
:param frame: Frame number of the scene that this merge task is
processing.
:type frame: int
:param task_id: Identifier of the task.
:type task_id: str
:param job_id: Unique identifier of the job. Job identifiers are unique
within a single Azure Batch account.
:type job_id: str
:param tile_num: Number of the current tile.
:type tile_num: int
:param current_x: X value of the current tile, used to generate the render
border.
:type current_x: int
:param current_y: Y value of the current tile, used to generate the render
border.
:type current_y: int
"""
blend_file = os.environ["BLEND_FILE"]
output_sas = os.environ["OUTPUT_CONTAINER_SAS"]
optionalParams = os.environ["OPTIONAL_PARAMS"]
command_line = blender_command(blend_file, optionalParams)
# only print this once
if task_id == 1:
print("tile task command line: {}".format(command_line))
return models.TaskAddParameter(
id=pad_number(task_id, PAD_LEN_ID),
display_name="frame: {}, tile: {}".format(frame, tile_num),
command_line=os_specific_command_line(command_line),
constraints=models.TaskConstraints(max_task_retry_count=2),
environment_settings=[
models.EnvironmentSetting("X_TILES", os.environ["X_TILES"]),
models.EnvironmentSetting("Y_TILES", os.environ["Y_TILES"]),
models.EnvironmentSetting("CROP_TO_BORDER",
os.environ["CROP_TO_BORDER"]),
models.EnvironmentSetting("OUTPUT_FORMAT",
os.environ["OUTPUT_FORMAT"]),
models.EnvironmentSetting("BLEND_FILE", os.environ["BLEND_FILE"]),
models.EnvironmentSetting("CURRENT_FRAME", str(frame)),
models.EnvironmentSetting("CURRENT_TILE", str(tile_num)),
models.EnvironmentSetting("CURRENT_X", str(current_x)),
models.EnvironmentSetting("CURRENT_Y", str(current_y))
],
resource_files=[
models.ResourceFile(
"https://raw.githubusercontent.com/Azure/BatchExplorer-data/master/ncj/blender/scripts/python-task-manager.py",
"scripts/python-task-manager.py")
],
output_files=[
models.OutputFile(
file_pattern="../stdout.txt",
destination=models.OutputFileDestination(
container=models.OutputFileBlobContainerDestination(
container_url=output_sas,
path="{}/logs/frame-{}/tile-{}.stdout.log".format(
job_id, pad_number(frame, PAD_LEN_FRAME),
pad_number(tile_num, PAD_LEN_TILE)))),
upload_options=models.OutputFileUploadOptions(
models.OutputFileUploadCondition.task_completion)),
models.OutputFile(
file_pattern="../stderr.txt",
destination=models.OutputFileDestination(
container=models.OutputFileBlobContainerDestination(
container_url=output_sas,
path="{}/logs/frame-{}/tile-{}.stderr.log".format(
job_id, pad_number(frame, PAD_LEN_FRAME),
pad_number(tile_num, PAD_LEN_TILE)))),
upload_options=models.OutputFileUploadOptions(
models.OutputFileUploadCondition.task_completion)),
models.OutputFile(
file_pattern="../fileuploaderr.txt",
destination=models.OutputFileDestination(
container=models.OutputFileBlobContainerDestination(
container_url=output_sas,
path="{}/logs/frame-{}/tile-{}.file_upload_stderr.log".
format(job_id, pad_number(frame, PAD_LEN_FRAME),
pad_number(tile_num, PAD_LEN_TILE)))),
upload_options=models.OutputFileUploadOptions(
models.OutputFileUploadCondition.task_completion)),
models.OutputFile(
file_pattern="../fileuploadout.txt",
destination=models.OutputFileDestination(
container=models.OutputFileBlobContainerDestination(
container_url=output_sas,
path="{}/logs/frame-{}/tile-{}.file_upload_stdout.log".
format(job_id, pad_number(frame, PAD_LEN_FRAME),
pad_number(tile_num, PAD_LEN_TILE)))),
upload_options=models.OutputFileUploadOptions(
models.OutputFileUploadCondition.task_completion)),
models.OutputFile(
file_pattern="tile_*",
destination=models.OutputFileDestination(
container=models.OutputFileBlobContainerDestination(
container_url=output_sas,
path="{}/outputs/frame-{}".format(
job_id, pad_number(frame, PAD_LEN_FRAME)))),
upload_options=models.OutputFileUploadOptions(
models.OutputFileUploadCondition.task_success))
])
def add_task(self, job_id: str, default_max_tries=None):
"""
Adds a task for each input file in the collection to the specified job.
:param str job_id: The ID of the job to which to add the tasks.
created for each input file.
:param int default_max_tries: Fallback max tries.
:output task: Azure Batch task
"""
from azure.batch import models as batchmodels
task_id = self.task_definition.get('id', job_id)
display_name = self.task_definition.get('displayName', task_id)
logging.info('Adding {} tasks to job [{}]...'.format(task_id, job_id))
container_settings = batchmodels.TaskContainerSettings(
image_name=self.image, container_run_options='--rm')
platform = self.conf[utils.PLATFORM]
environment_settings = [
batchmodels.EnvironmentSetting(name='AZURE_SUBSCRIPTION_ID',
value=platform['subscription']),
batchmodels.EnvironmentSetting(name='AZURE_STORAGE_ACCOUNT',
value=platform['storage_account']),
batchmodels.EnvironmentSetting(
name='AZURE_STORAGE_CONTAINER',
value=platform['storage_container']),
batchmodels.EnvironmentSetting(
name='AZURE_STORAGE_CONNECTION_STRING',
value=platform['storage_connection_string']),
]
if 'environmentSettings' in self.task_definition and self.task_definition[
'environmentSettings'] is not None:
environment_settings.extend([
batchmodels.EnvironmentSetting(**setting)
for setting in self.task_definition['environmentSettings']
])
constraints = None
if 'constraints' in self.task_definition and self.task_definition[
'constraints']:
constraints = batchmodels.TaskConstraints(
max_wall_clock_time=self.task_definition['constraints'].get(
'maxWallClockTime', "P1D"),
max_task_retry_count=self.task_definition['constraints'].get(
'maxTaskRetryCount', default_max_tries),
retention_time=self.task_definition['constraints'].get(
'retentionTime', "P1D"),
),
user_identity = batchmodels.UserIdentity(
auto_user=batchmodels.AutoUserSpecification(
scope=batchmodels.AutoUserScope.pool,
elevation_level=batchmodels.ElevationLevel.admin))
task = batchmodels.TaskAddParameter(
id=task_id,
display_name=display_name,
command_line=self.task_definition['commandLine'],
constraints=constraints[0],
container_settings=container_settings,
environment_settings=environment_settings,
user_identity=user_identity,
)
for validation in task.validate():
logging.info(validation)
self.batch_client.task.add(job_id=job_id, task=task)
return task
def generate_task(spark_client, container_id, application, remote=False):
resource_files = []
# The application provided is not hosted remotely and therefore must be uploaded
if not remote:
app_resource_file = helpers.upload_file_to_container(
container_name=container_id,
application_name=application.name,
file_path=application.application,
blob_client=spark_client.blob_client,
use_full_path=False,
)
# Upload application file
resource_files.append(app_resource_file)
application.application = "$AZ_BATCH_TASK_WORKING_DIR/" + os.path.basename(
application.application)
# Upload dependent JARS
jar_resource_file_paths = []
for jar in application.jars:
current_jar_resource_file_path = helpers.upload_file_to_container(
container_name=container_id,
application_name=application.name,
file_path=jar,
blob_client=spark_client.blob_client,
use_full_path=False,
)
jar_resource_file_paths.append(current_jar_resource_file_path)
resource_files.append(current_jar_resource_file_path)
# Upload dependent python files
py_files_resource_file_paths = []
for py_file in application.py_files:
current_py_files_resource_file_path = helpers.upload_file_to_container(
container_name=container_id,
application_name=application.name,
file_path=py_file,
blob_client=spark_client.blob_client,
use_full_path=False,
)
py_files_resource_file_paths.append(
current_py_files_resource_file_path)
resource_files.append(current_py_files_resource_file_path)
# Upload other dependent files
files_resource_file_paths = []
for file in application.files:
files_resource_file_path = helpers.upload_file_to_container(
container_name=container_id,
application_name=application.name,
file_path=file,
blob_client=spark_client.blob_client,
use_full_path=False,
)
files_resource_file_paths.append(files_resource_file_path)
resource_files.append(files_resource_file_path)
# Upload application definition
application.jars = [os.path.basename(jar) for jar in application.jars]
application.py_files = [
os.path.basename(py_files) for py_files in application.py_files
]
application.files = [
os.path.basename(files) for files in application.files
]
application_definition_file = helpers.upload_text_to_container(
container_name=container_id,
application_name=application.name,
file_path="application.yaml",
content=yaml.dump(vars(application)),
blob_client=spark_client.blob_client,
)
resource_files.append(application_definition_file)
# create command to submit task
task_cmd = CommandBuilder("sudo docker exec")
task_cmd.add_argument("-i")
task_cmd.add_option(
"-e", "AZ_BATCH_TASK_WORKING_DIR=$AZ_BATCH_TASK_WORKING_DIR")
task_cmd.add_option("-e",
"STORAGE_LOGS_CONTAINER={0}".format(container_id))
task_cmd.add_argument("spark /bin/bash >> output.log 2>&1")
task_cmd.add_argument(
r'-c "source ~/.bashrc; '
r"export PYTHONPATH=$PYTHONPATH:\$AZTK_WORKING_DIR; "
r"cd \$AZ_BATCH_TASK_WORKING_DIR; "
r'\$AZTK_WORKING_DIR/.aztk-env/.venv/bin/python \$AZTK_WORKING_DIR/aztk/node_scripts/submit.py"'
)
# Create task
task = batch_models.TaskAddParameter(
id=application.name,
command_line=helpers.wrap_commands_in_shell([task_cmd.to_str()]),
resource_files=resource_files,
constraints=batch_models.TaskConstraints(
max_task_retry_count=application.max_retry_count),
user_identity=batch_models.UserIdentity(
auto_user=batch_models.AutoUserSpecification(
scope=batch_models.AutoUserScope.task,
elevation_level=batch_models.ElevationLevel.admin)),
)
return task
def generate_task(spark_client, container_id, application):
resource_files = []
app_resource_file = helpers.upload_file_to_container(
container_name=container_id,
application_name=application.name,
file_path=application.application,
blob_client=spark_client.blob_client,
use_full_path=False)
# Upload application file
resource_files.append(app_resource_file)
# Upload dependent JARS
jar_resource_file_paths = []
for jar in application.jars:
current_jar_resource_file_path = helpers.upload_file_to_container(
container_name=container_id,
application_name=application.name,
file_path=jar,
blob_client=spark_client.blob_client,
use_full_path=False)
jar_resource_file_paths.append(current_jar_resource_file_path)
resource_files.append(current_jar_resource_file_path)
# Upload dependent python files
py_files_resource_file_paths = []
for py_file in application.py_files:
current_py_files_resource_file_path = helpers.upload_file_to_container(
container_name=container_id,
application_name=application.name,
file_path=py_file,
blob_client=spark_client.blob_client,
use_full_path=False)
py_files_resource_file_paths.append(
current_py_files_resource_file_path)
resource_files.append(current_py_files_resource_file_path)
# Upload other dependent files
files_resource_file_paths = []
for file in application.files:
files_resource_file_path = helpers.upload_file_to_container(
container_name=container_id,
application_name=application.name,
file_path=file,
blob_client=spark_client.blob_client,
use_full_path=False)
files_resource_file_paths.append(files_resource_file_path)
resource_files.append(files_resource_file_path)
# Upload application definition
application.application = os.path.basename(application.application)
application.jars = [os.path.basename(jar) for jar in application.jars]
application.py_files = [
os.path.basename(py_files) for py_files in application.py_files
]
application.files = [
os.path.basename(files) for files in application.files
]
application_definition_file = helpers.upload_text_to_container(
container_name=container_id,
application_name=application.name,
file_path='application.yaml',
content=yaml.dump(vars(application)),
blob_client=spark_client.blob_client)
resource_files.append(application_definition_file)
# create command to submit task
task_cmd = CommandBuilder('sudo docker exec')
task_cmd.add_argument('-i')
task_cmd.add_option(
'-e', 'AZ_BATCH_TASK_WORKING_DIR=$AZ_BATCH_TASK_WORKING_DIR')
task_cmd.add_option('-e',
'STORAGE_LOGS_CONTAINER={0}'.format(container_id))
task_cmd.add_argument('spark /bin/bash >> output.log 2>&1')
task_cmd.add_argument('-c "source ~/.bashrc; '\
'cd $AZ_BATCH_TASK_WORKING_DIR; ' \
'\$(pyenv root)/versions/\$AZTK_PYTHON_VERSION/bin/python ' \
'\$DOCKER_WORKING_DIR/aztk/node_scripts/submit.py"')
# Create task
task = batch_models.TaskAddParameter(
id=application.name,
command_line=helpers.wrap_commands_in_shell([task_cmd.to_str()]),
resource_files=resource_files,
constraints=batch_models.TaskConstraints(
max_task_retry_count=application.max_retry_count),
user_identity=batch_models.UserIdentity(
auto_user=batch_models.AutoUserSpecification(
scope=batch_models.AutoUserScope.task,
elevation_level=batch_models.ElevationLevel.admin)))
return task
def add_task(batch_service_client, job_id, input_files,
output_container_sas_url, task_name, input_file_dir):
"""
Adds a task for each input file in the collection to the specified job.
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str job_id: The ID of the job to which to add the tasks.
:param list input_files: A collection of input files. One task will be
created for each input file.
:param output_container_sas_token: A SAS token granting write access to
the specified Azure Blob storage container.
"""
tasks = list()
# my own tasks section
logging.info('Adding dashcam task [{task}] to job [{job}]...'.format(
task=task_name, job=job_id))
output_file_path = "/mnt/batch/tasks/workitems/{jobname}/job-1/{taskname}/wd".format(
jobname=job_id, taskname=task_name)
logging.info('input_file_dir: {}'.format(input_file_dir))
logging.info('output_file_path: {}'.format(output_file_path))
# Command uses the script from https://github.com/ehendrix23/tesla_dashcam
command = '''tesla_dashcam --no-timestamp --monitor_trigger {input} --output {output} --motion_only --mirror --layout DIAMOND --fps 33 --quality HIGH --slowdown 5'''.format(
input=input_file_dir, output=output_file_path)
logging.info('command: {}'.format(command))
tasks.append(
batch.models.TaskAddParameter(
id=task_name,
command_line=command,
constraints=batchmodels.TaskConstraints(
max_wall_clock_time=datetime.timedelta(minutes=60),
max_task_retry_count=1),
resource_files=input_files,
output_files=[
batchmodels.OutputFile(
file_pattern='*.mp4',
destination=batchmodels.OutputFileDestination(
container=batchmodels.
OutputFileBlobContainerDestination(
path='',
container_url=
output_container_sas_url #os.environ["_SAS"]
)),
upload_options=batchmodels.OutputFileUploadOptions(
upload_condition=batchmodels.OutputFileUploadCondition.
task_success)),
batchmodels.OutputFile(
file_pattern='../std*.txt',
destination=batchmodels.OutputFileDestination(
container=batchmodels.
OutputFileBlobContainerDestination(
path=task_name,
container_url=
output_container_sas_url #os.environ["_SAS"]
)),
upload_options=batchmodels.OutputFileUploadOptions(
upload_condition=batchmodels.OutputFileUploadCondition.
task_success)),
batchmodels.OutputFile(
file_pattern='../fileupload*.txt',
destination=batchmodels.OutputFileDestination(
container=batchmodels.
OutputFileBlobContainerDestination(
path=task_name,
container_url=
output_container_sas_url #os.environ["_SAS"]
)),
upload_options=batchmodels.OutputFileUploadOptions(
upload_condition=batchmodels.OutputFileUploadCondition.
task_success))
]))
batch_service_client.task.add_collection(job_id, tasks)
def submit_application(spark_client, cluster_id, application, wait: bool = False):
"""
Submit a spark app
"""
resource_files = []
app_resource_file = helpers.upload_file_to_container(container_name=application.name,
file_path=application.application,
blob_client=spark_client.blob_client,
use_full_path=False)
# Upload application file
resource_files.append(app_resource_file)
# Upload dependent JARS
jar_resource_file_paths = []
for jar in application.jars:
current_jar_resource_file_path = helpers.upload_file_to_container(container_name=application.name,
file_path=jar,
blob_client=spark_client.blob_client,
use_full_path=False)
jar_resource_file_paths.append(current_jar_resource_file_path)
resource_files.append(current_jar_resource_file_path)
# Upload dependent python files
py_files_resource_file_paths = []
for py_file in application.py_files:
current_py_files_resource_file_path = helpers.upload_file_to_container(container_name=application.name,
file_path=py_file,
blob_client=spark_client.blob_client,
use_full_path=False)
py_files_resource_file_paths.append(
current_py_files_resource_file_path)
resource_files.append(current_py_files_resource_file_path)
# Upload other dependent files
files_resource_file_paths = []
for file in application.files:
files_resource_file_path = helpers.upload_file_to_container(container_name=application.name,
file_path=file,
blob_client=spark_client.blob_client,
use_full_path=False)
files_resource_file_paths.append(files_resource_file_path)
resource_files.append(files_resource_file_path)
# create command to submit task
cmd = __app_submit_cmd(
spark_client=spark_client,
cluster_id=cluster_id,
name=application.name,
app=app_resource_file.file_path,
app_args=application.application_args,
main_class=application.main_class,
jars=[jar_resource_file_path.file_path for jar_resource_file_path in jar_resource_file_paths],
py_files=[py_files_resource.file_path for py_files_resource in py_files_resource_file_paths],
files=[file_resource_file_path.file_path for file_resource_file_path in files_resource_file_paths],
driver_java_options=application.driver_java_options,
driver_library_path=application.driver_library_path,
driver_class_path=application.driver_class_path,
driver_memory=application.driver_memory,
executor_memory=application.executor_memory,
driver_cores=application.driver_cores,
executor_cores=application.executor_cores)
# Get cluster size
cluster = spark_client.get_cluster(cluster_id)
# Affinitize task to master node
# master_node_affinity_id = helpers.get_master_node_id(cluster_id, spark_client.batch_client)
rls = spark_client.get_remote_login_settings(cluster.id, cluster.master_node_id)
# Create task
task = batch_models.TaskAddParameter(
id=application.name,
affinity_info=batch_models.AffinityInformation(
affinity_id=cluster.master_node_id),
command_line=helpers.wrap_commands_in_shell(cmd),
resource_files=resource_files,
constraints=batch_models.TaskConstraints(
max_task_retry_count=application.max_retry_count),
user_identity=batch_models.UserIdentity(
auto_user=batch_models.AutoUserSpecification(
scope=batch_models.AutoUserScope.task,
elevation_level=batch_models.ElevationLevel.admin))
)
# Add task to batch job (which has the same name as cluster_id)
job_id = cluster_id
spark_client.batch_client.task.add(job_id=job_id, task=task)
if wait:
helpers.wait_for_task_to_complete(job_id=job_id, task_id=task.id, batch_client=spark_client.batch_client)
def create_task(dataset, command, dependencies, max_wall_clock, production):
if production:
container = RESULTS_CONTAINER
else:
container = TEST_RESULTS_CONTAINER + "/" + \
generate_task_name(dataset.name)
output_files = [
# Upload results
batch_models.OutputFile(
file_pattern="$AZ_BATCH_TASK_WORKING_DIR/results/**/*",
upload_options=batch_models.OutputFileUploadOptions(
upload_condition=batch_models.OutputFileUploadCondition.
task_success),
destination=batch_models.OutputFileDestination(
container=batch_models.OutputFileBlobContainerDestination(
path=dataset.data_dir, container_url=container +
SAS_TOKEN))),
batch_models.OutputFile(
file_pattern=
f"$AZ_BATCH_NODE_ROOT_DIR/fsmounts/{FILE_SHARE_NAME}/*.csv",
upload_options=batch_models.OutputFileUploadOptions(
upload_condition=batch_models.OutputFileUploadCondition.
task_success),
destination=batch_models.OutputFileDestination(
container=batch_models.OutputFileBlobContainerDestination(
container_url=container + SAS_TOKEN))),
batch_models.OutputFile(
file_pattern=
f"$AZ_BATCH_NODE_ROOT_DIR/fsmounts/{FILE_SHARE_NAME}/last-update/*",
upload_options=batch_models.OutputFileUploadOptions(
upload_condition=batch_models.OutputFileUploadCondition.
task_success),
destination=batch_models.OutputFileDestination(
container=batch_models.OutputFileBlobContainerDestination(
path="last-update", container_url=container + SAS_TOKEN))),
# Upload stderr and stdout
batch_models.OutputFile(
file_pattern="$AZ_BATCH_TASK_DIR/std*.txt",
upload_options=batch_models.OutputFileUploadOptions(
upload_condition=batch_models.OutputFileUploadCondition.
task_completion),
destination=batch_models.OutputFileDestination(
container=batch_models.OutputFileBlobContainerDestination(
path=DATETIME_NOWISH + "/" +
generate_task_name(dataset.name),
container_url=PROCESS_LOG_CONTAINER + "/" + SAS_TOKEN)))
]
return batch_models.TaskAddParameter(
id=generate_task_name(dataset.name),
display_name=(dataset.name + "_python_script_job"),
command_line=command,
resource_files=[
batch_models.ResourceFile(storage_container_url=CONFIG_CONTAINER +
SAS_TOKEN,
blob_prefix=dataset.name + CONFIG_FILE)
],
depends_on=batch_models.TaskDependencies(task_ids=dependencies),
user_identity=batch_models.UserIdentity(
auto_user=batch_models.AutoUserSpecification(
scope='pool', elevation_level='admin')),
container_settings=batch_models.TaskContainerSettings(
image_name=DOCKER_CONTAINER_URL,
container_run_options='-w /home/rstudio/covid-rt-estimates'),
constraints=batch_models.TaskConstraints(
max_wall_clock_time=datetime.timedelta(minutes=max_wall_clock)),
output_files=output_files)
