def create_task(client,
job_id,
json_file=None,
task_id=None,
command_line=None,
resource_files=None,
environment_settings=None,
affinity_info=None,
max_wall_clock_time=None,
retention_time=None,
max_task_retry_count=None,
application_package_references=None):
def action():
if task is not None:
client.add(job_id=job_id, task=task)
return client.get(job_id=job_id, task_id=task.id)
submitted_tasks = []
for i in range(0, len(tasks), MAX_TASKS_PER_REQUEST):
submission = client.add_collection(
job_id=job_id, value=tasks[i:i + MAX_TASKS_PER_REQUEST])
submitted_tasks.extend(submission.value) # pylint: disable=no-member
return submitted_tasks
task = None
tasks = []
if json_file:
with open(json_file) as f:
json_obj = json.load(f)
try:
task = client._deserialize( # pylint: disable=protected-access
'TaskAddParameter', json_obj)
except DeserializationError:
try:
tasks = client._deserialize( # pylint: disable=protected-access
'[TaskAddParameter]', json_obj)
except DeserializationError:
raise ValueError(
"JSON file '{}' is not formatted correctly.".format(
json_file))
else:
if command_line is None or task_id is None:
raise ValueError(
"Missing required arguments.\nEither --json-file, "
"or both --task-id and --command-line must be specified.")
task = TaskAddParameter(
task_id,
command_line,
resource_files=resource_files,
environment_settings=environment_settings,
affinity_info=affinity_info,
application_package_references=application_package_references)
if max_wall_clock_time is not None or retention_time is not None \
or max_task_retry_count is not None:
task.constraints = TaskConstraints(
max_wall_clock_time=max_wall_clock_time,
retention_time=retention_time,
max_task_retry_count=max_task_retry_count)
return _handle_batch_exception(action)
def create_task(client,
job_id,
json_file=None,
task_id=None,
command_line=None,
resource_files=None,
environment_settings=None,
affinity_id=None,
max_wall_clock_time=None,
retention_time=None,
max_task_retry_count=None,
application_package_references=None):
task = None
tasks = []
if json_file:
json_obj = get_file_json(json_file)
try:
task = TaskAddParameter.from_dict(json_obj)
except DeserializationError:
tasks = []
try:
for json_task in json_obj:
tasks.append(TaskAddParameter.from_dict(json_task))
except (DeserializationError, TypeError):
raise ValueError(
"JSON file '{}' is not formatted correctly.".format(
json_file))
else:
if command_line is None or task_id is None:
raise ValueError(
"Missing required arguments.\nEither --json-file, "
"or both --task-id and --command-line must be specified.")
task = TaskAddParameter(
task_id,
command_line,
resource_files=resource_files,
environment_settings=environment_settings,
affinity_info=AffinityInformation(affinity_id)
if affinity_id else None,
application_package_references=application_package_references)
if max_wall_clock_time is not None or retention_time is not None \
or max_task_retry_count is not None:
task.constraints = TaskConstraints(
max_wall_clock_time=max_wall_clock_time,
retention_time=retention_time,
max_task_retry_count=max_task_retry_count)
if task is not None:
client.add(job_id=job_id, task=task)
return client.get(job_id=job_id, task_id=task.id)
submitted_tasks = []
for i in range(0, len(tasks), MAX_TASKS_PER_REQUEST):
submission = client.add_collection(job_id=job_id,
value=tasks[i:i +
MAX_TASKS_PER_REQUEST])
submitted_tasks.extend(submission.value) # pylint: disable=no-member
return submitted_tasks
def create_task(
client,
job_id,
json_file=None,
task_id=None,
command_line=None, # pylint:disable=too-many-arguments
resource_files=None,
environment_settings=None,
affinity_info=None,
max_wall_clock_time=None,
retention_time=None,
max_task_retry_count=None,
application_package_references=None):
def action():
if task is not None:
client.add(job_id=job_id, task=task)
return client.get(job_id=job_id, task_id=task.id)
else:
result = client.add_collection(job_id=job_id, value=tasks)
return result.value
task = None
if json_file:
with open(json_file) as f:
json_obj = json.load(f)
try:
task = client._deserialize('TaskAddParameter', json_obj) # pylint: disable=protected-access
except DeserializationError:
try:
tasks = client._deserialize('[TaskAddParameter]', json_obj) # pylint: disable=protected-access
except DeserializationError:
raise ValueError(
"JSON file '{}' is not in reqired format.".format(
json_file))
else:
if command_line is None or task_id is None:
raise ValueError(
"Missing required arguments.\nEither --json-file, "
"or both --task-id and --command-line must be specified.")
task = TaskAddParameter(
task_id,
command_line,
resource_files=resource_files,
environment_settings=environment_settings,
affinity_info=affinity_info,
application_package_references=application_package_references)
if max_wall_clock_time is not None or retention_time is not None \
or max_task_retry_count is not None:
task.constraints = TaskConstraints(
max_wall_clock_time=max_wall_clock_time,
retention_time=retention_time,
max_task_retry_count=max_task_retry_count)
return _handle_batch_exception(action)
def create_task(
client,
job_id,
json_file=None,
task_id=None,
command_line=None, # pylint:disable=too-many-arguments
resource_files=None,
environment_settings=None,
affinity_info=None,
max_wall_clock_time=None,
retention_time=None,
max_task_retry_count=None,
run_elevated=None,
application_package_references=None,
):
def action():
if task is not None:
client.add(job_id=job_id, task=task)
return client.get(job_id=job_id, task_id=task.id)
else:
result = client.add_collection(job_id=job_id, value=tasks)
return result.value
task = None
if json_file:
with open(json_file) as f:
json_obj = json.load(f)
try:
task = client._deserialize("TaskAddParameter", json_obj) # pylint: disable=protected-access
except DeserializationError:
try:
tasks = client._deserialize("[TaskAddParameter]", json_obj) # pylint: disable=protected-access
except DeserializationError:
raise ValueError("JSON file '{}' is not in reqired format.".format(json_file))
else:
task = TaskAddParameter(
task_id,
command_line,
resource_files=resource_files,
environment_settings=environment_settings,
affinity_info=affinity_info,
run_elevated=run_elevated,
application_package_references=application_package_references,
)
if max_wall_clock_time is not None or retention_time is not None or max_task_retry_count is not None:
task.constraints = TaskConstraints(
max_wall_clock_time=max_wall_clock_time,
retention_time=retention_time,
max_task_retry_count=max_task_retry_count,
)
return _handle_batch_exception(action)
def create_task(client,
job_id, json_file=None, task_id=None, command_line=None, resource_files=None,
environment_settings=None, affinity_id=None, max_wall_clock_time=None,
retention_time=None, max_task_retry_count=None,
application_package_references=None):
task = None
tasks = []
if json_file:
json_obj = get_file_json(json_file)
try:
task = TaskAddParameter.from_dict(json_obj)
except (DeserializationError, TypeError):
try:
task_collection = TaskAddCollectionParameter.from_dict(json_obj)
tasks = task_collection.value
except (DeserializationError, TypeError):
try:
for json_task in json_obj:
tasks.append(TaskAddParameter.from_dict(json_task))
except (DeserializationError, TypeError):
raise ValueError("JSON file '{}' is not formatted correctly.".format(json_file))
else:
if command_line is None or task_id is None:
raise ValueError("Missing required arguments.\nEither --json-file, "
"or both --task-id and --command-line must be specified.")
task = TaskAddParameter(
id=task_id,
command_line=command_line,
resource_files=resource_files,
environment_settings=environment_settings,
affinity_info=AffinityInformation(affinity_id=affinity_id) if affinity_id else None,
application_package_references=application_package_references)
if max_wall_clock_time is not None or retention_time is not None \
or max_task_retry_count is not None:
task.constraints = TaskConstraints(max_wall_clock_time=max_wall_clock_time,
retention_time=retention_time,
max_task_retry_count=max_task_retry_count)
if task is not None:
client.add(job_id=job_id, task=task)
return client.get(job_id=job_id, task_id=task.id)
submitted_tasks = []
for i in range(0, len(tasks), MAX_TASKS_PER_REQUEST):
submission = client.add_collection(
job_id=job_id,
value=tasks[i:i + MAX_TASKS_PER_REQUEST])
submitted_tasks.extend(submission.value) # pylint: disable=no-member
return submitted_tasks
def create_task(client,
job_id, json_file=None, task_id=None, command_line=None, resource_files=None,
environment_settings=None, affinity_info=None, max_wall_clock_time=None,
retention_time=None, max_task_retry_count=None,
application_package_references=None):
def action():
if task is not None:
client.add(job_id=job_id, task=task)
return client.get(job_id=job_id, task_id=task.id)
result = client.add_collection(job_id=job_id, value=tasks)
return result.value
task = None
if json_file:
with open(json_file) as f:
json_obj = json.load(f)
try:
task = client._deserialize( # pylint: disable=protected-access
'TaskAddParameter', json_obj)
except DeserializationError:
try:
tasks = client._deserialize( # pylint: disable=protected-access
'[TaskAddParameter]', json_obj)
except DeserializationError:
raise ValueError("JSON file '{}' is not in reqired format.".format(json_file))
else:
if command_line is None or task_id is None:
raise ValueError("Missing required arguments.\nEither --json-file, "
"or both --task-id and --command-line must be specified.")
task = TaskAddParameter(task_id, command_line,
resource_files=resource_files,
environment_settings=environment_settings,
affinity_info=affinity_info,
application_package_references=application_package_references)
if max_wall_clock_time is not None or retention_time is not None \
or max_task_retry_count is not None:
task.constraints = TaskConstraints(max_wall_clock_time=max_wall_clock_time,
retention_time=retention_time,
max_task_retry_count=max_task_retry_count)
return _handle_batch_exception(action)
def create_build_job(commit_sha: str) -> CloudJob:
"""
Schedule a build job in the given pool. returns the container for build output and job reference.
Building and running tests are two separate builds so that the testing job can relies on job preparation tasks to
prepare test environment. The product and test build is an essential part of the preparation. The builds can't be
combined because the preparation task has to be defined by the time the job is created. However neither the product
or the test package is ready then.
"""
batch_client = get_batch_client()
source_control_info = get_source_control_info()
remote_source_dir = 'gitsrc'
logger = get_logger('build')
pool = get_batch_pool('build')
if not pool:
logger.error(
'Cannot find a build pool. Please check the pools list in config file.'
)
raise ValueError('Fail to find a build pool.')
# secret is a random string used to verify the identity of caller when one task requests the service to do
# something. the secret is saved to the job definition as metadata, and it is passed to some tasks as well.
secret = base64.b64encode(os.urandom(64)).decode('utf-8')
job_metadata = [
MetadataItem('usage', 'build'),
MetadataItem('secret', secret),
MetadataItem('source_url', source_control_info.url),
MetadataItem('source_sha', commit_sha)
]
logger.info('Creating build job %s in pool %s', commit_sha, pool.id)
batch_client.job.add(
JobAddParameter(id=commit_sha,
pool_info=PoolInformation(pool.id),
on_all_tasks_complete=OnAllTasksComplete.terminate_job,
metadata=job_metadata,
uses_task_dependencies=True))
logger.info('Job %s is created.', commit_sha)
output_file_name = 'azure-cli-{}.tar'.format(commit_sha)
build_commands = [
'git clone --depth=50 {} {}'.format(source_control_info.url,
remote_source_dir),
'pushd {}'.format(remote_source_dir),
'git checkout -qf {}'.format(commit_sha),
'./scripts/batch/build_all.sh',
'tar cvzf {} ./artifacts'.format(output_file_name)
]
build_container_url = _get_build_blob_container_url()
output_file = OutputFile(
'{}/{}'.format(remote_source_dir, output_file_name),
OutputFileDestination(
OutputFileBlobContainerDestination(build_container_url,
output_file_name)),
OutputFileUploadOptions(OutputFileUploadCondition.task_success))
build_task = TaskAddParameter(
id='build',
command_line=get_command_string(*build_commands),
display_name='Build all product and test code.',
output_files=[output_file])
cburl = 'curl -X post {} -H "X-Batch-Event: build.finished" --data-urlencode secret={} --data-urlencode sha={}'
report_cmd = cburl.format(
url_for('post_api_build', _external=True, _scheme='https'), secret,
commit_sha)
report_task = TaskAddParameter(
id='report',
command_line=get_command_string(report_cmd),
depends_on=TaskDependencies(task_ids=[build_task.id]),
display_name='Request service to pull result')
batch_client.task.add(commit_sha, build_task)
batch_client.task.add(commit_sha, report_task)
logger.info('Build task is added to job %s', commit_sha)
return batch_client.job.get(commit_sha)
def start_worker_job(self) -> None:
worker_image: str = 'codalab/worker:' + os.environ.get('CODALAB_VERSION', 'latest')
worker_id: str = uuid.uuid4().hex
logger.debug('Starting worker {} with image {}'.format(worker_id, worker_image))
work_dir_prefix: str = (
self.args.worker_work_dir_prefix if self.args.worker_work_dir_prefix else "/tmp/"
)
# This needs to be a unique directory since Batch jobs may share a host
work_dir: str = os.path.join(work_dir_prefix, 'cl_worker_{}_work_dir'.format(worker_id))
command: List[str] = self.build_command(worker_id, work_dir)
task_container_run_options: List[str] = [
'--cpus %d' % self.args.cpus,
'--memory %dM' % self.args.memory_mb,
'--volume /var/run/docker.sock:/var/run/docker.sock',
'--volume %s:%s' % (work_dir, work_dir),
'--user %s' % self.args.user,
]
if os.environ.get('CODALAB_USERNAME') and os.environ.get('CODALAB_PASSWORD'):
task_container_run_options.extend(
[
'--env CODALAB_USERNAME=%s' % os.environ.get('CODALAB_USERNAME'),
'--env CODALAB_PASSWORD=%s' % os.environ.get('CODALAB_PASSWORD'),
]
)
else:
raise EnvironmentError(
'Valid credentials need to be set as environment variables: CODALAB_USERNAME and CODALAB_PASSWORD'
)
if os.environ.get('CODALAB_SHARED_FILE_SYSTEM') == 'true':
# Allow workers to directly mount a directory
command.append('--shared-file-system')
task_container_run_options.append(
'--volume shared_dir:%s' % os.environ.get('CODALAB_BUNDLE_MOUNT')
)
# Configure Sentry
if using_sentry():
task_container_run_options.append(
'--env CODALAB_SENTRY_INGEST_URL=%s' % CODALAB_SENTRY_INGEST
)
task_container_run_options.append(
'--env CODALAB_SENTRY_ENVIRONMENT=%s' % CODALAB_SENTRY_ENVIRONMENT
)
command_line: str = "/bin/bash -c '{}'".format(' '.join(command))
logger.debug("Running the following as an Azure Batch task: {}".format(command_line))
task_id: str = 'cl_worker_{}'.format(worker_id)
task: TaskAddParameter = TaskAddParameter(
id=task_id,
command_line=command_line,
container_settings=TaskContainerSettings(
image_name=worker_image, container_run_options=' '.join(task_container_run_options)
),
output_files=[
OutputFile(
file_pattern='../stderr.txt',
destination=OutputFileDestination(
container=OutputFileBlobContainerDestination(
path=task_id, container_url=self.args.log_container_url
)
),
upload_options=OutputFileUploadOptions(
# Upload worker logs once the task completes
upload_condition=OutputFileUploadCondition.task_completion
),
)
],
)
try:
# Create a task under the Azure Batch job.
# Catch request errors to keep the worker manager running.
self._batch_client.task.add(self.args.job_id, task)
except (ClientRequestError, BatchErrorException) as e:
logger.error(
'Batch request to add task {} to job {} failed: {}'.format(
task_id, self.args.job_id, str(e)
)
)
