def test_submit_validator_passes(self):
job_params = {'inputs': set(), 'outputs': set(), 'mounts': set()}
job_resources = job_model.Resources(
logging=job_model.LoggingParam('gs://buck/logs', job_model.P_GCS))
param_util.validate_submit_args_or_fail(
job_model.JobDescriptor(None, job_params, job_resources,
TASK_DESCRIPTORS),
provider_name='MYPROVIDER',
input_providers=[job_model.P_GCS],
output_providers=[job_model.P_GCS],
logging_providers=[job_model.P_GCS])
def test_submit_validator_fails(self, name, path, inwl, outwl, logwl):
job_params = {'inputs': set(), 'outputs': set(), 'mounts': set()}
job_resources = job_model.Resources(
logging=job_model.LoggingParam('gs://buck/logs', job_model.P_GCS))
err_expected = 'Unsupported %s path (%s) for provider' % (name, path)
with six.assertRaisesRegex(self, ValueError, re.escape(err_expected)):
param_util.validate_submit_args_or_fail(job_model.JobDescriptor(
None, job_params, job_resources, TASK_DESCRIPTORS),
provider_name='MYPROVIDER',
input_providers=inwl,
output_providers=outwl,
logging_providers=logwl)
def test_foo(self, retries, max_preemptible_attempts,
list_of_list_of_operations, expected):
def chronology(list_of_list_of_operations):
for operations in list_of_list_of_operations:
self.provider.set_operations(operations)
yield 1
establish_chronology(chronology(list_of_list_of_operations))
job_descriptor = job_model.JobDescriptor(
job_metadata={'create-time': 123456},
job_params=None,
job_resources=job_model.Resources(
logging=job_model.LoggingParam('gs://buck/logs', job_model.P_GCS),
max_preemptible_attempts=param_util.PreemptibleParam(
max_preemptible_attempts)),
task_descriptors=[
job_model.TaskDescriptor(
task_metadata={
'task-id': 123,
'create-time': 123456
},
task_params=None,
task_resources=None)
])
poll_interval = 1
ret = dsub_command._wait_and_retry(
self.provider,
'job-1',
poll_interval,
retries,
job_descriptor,
summary=False)
tasks = self.provider.lookup_job_tasks({'*'})
# First, the number of tasks returned by lookup_job_tasks should be equal
# to the total number of task attempts.
expected_num_of_tasks = len(list_of_list_of_operations[-1])
self.assertEqual(len(tasks), expected_num_of_tasks)
# Second, check that the return value of _wait_and_retry is correct.
self.assertEqual(ret, expected)
def dsub_start_job(command,
job_name=None,
envs=None,
labels=None,
inputs=None,
inputs_recursive=None,
outputs=None,
outputs_recursive=None,
wait=False):
envs = envs or {}
labels = labels or {}
inputs = inputs or {}
inputs_recursive = inputs_recursive or {}
outputs = outputs or {}
outputs_recursive = outputs_recursive or {}
labels['test-token'] = test_setup.TEST_TOKEN
labels['test-name'] = test_setup.TEST_NAME
logging = param_util.build_logging_param(test.LOGGING)
job_resources = job_model.Resources(image='ubuntu',
logging=logging,
zones=['us-central1-*'])
env_data = {job_model.EnvParam(k, v) for (k, v) in envs.items()}
label_data = {job_model.LabelParam(k, v) for (k, v) in labels.items()}
input_file_param_util = param_util.InputFileParamUtil('input')
input_data = set()
for (recursive, items) in ((False, inputs.items()),
(True, inputs_recursive.items())):
for (name, value) in items:
name = input_file_param_util.get_variable_name(name)
input_data.add(
input_file_param_util.make_param(name, value, recursive))
output_file_param_util = param_util.OutputFileParamUtil('output')
output_data = set()
for (recursive, items) in ((False, outputs.items()),
(True, outputs_recursive.items())):
for (name, value) in items:
name = output_file_param_util.get_variable_name(name)
output_data.add(
output_file_param_util.make_param(name, value, recursive))
job_params = {
'envs': env_data,
'inputs': input_data,
'outputs': output_data,
'labels': label_data,
}
task_descriptors = [
job_model.TaskDescriptor({'task-id': None}, {
'envs': set(),
'labels': set(),
'inputs': set(),
'outputs': set(),
}, job_model.Resources())
]
return dsub.run(get_dsub_provider(),
job_resources,
job_params,
task_descriptors,
name=job_name,
command=command,
wait=wait,
disable_warning=True)
logging: gs://b/dsub/sh/local/env_list/env_list/logging/env_list.log
tasks:
- logging-path: gs://b/dsub/sh/local/env_list/env_list/logging/env_list.log
task-id: null
user-id: dsubuser
""")
_ENV_LIST_JOB_DESCRIPTOR = job_model.JobDescriptor(
job_metadata={
'dsub-version': '0.1.4',
'job-id': 'script_env--dsubuser--171122-142837-321721',
'job-name': 'script_env_test.sh',
'user-id': 'dsubuser',
'create-time': CREATE_TIME
},
job_resources=job_model.Resources(
logging='gs://b/dsub/sh/local/env_list/env_list/logging/env_list.log'),
job_params={
'envs': {
job_model.EnvParam('VAR1', 'VAL1'),
job_model.EnvParam('VAR2', 'VAL2'),
job_model.EnvParam('VAR3', 'VAL3'),
job_model.EnvParam('VAR4', 'VAL4'),
job_model.EnvParam('VAR5', 'VAL5'),
},
'labels': set(),
'inputs': set(),
'outputs': set(),
'mounts': set(),
},
task_descriptors=[
job_model.TaskDescriptor(
def start_job(self,
command,
name=None,
envs={},
labels={},
inputs={},
inputs_recursive={},
outputs={},
outputs_recursive={},
task_count=1,
wait=False):
logging = param_util.build_logging_param(self.log_path)
resources = job_model.Resources(image=DOCKER_IMAGE,
logging=logging,
zones=['us-central1*'])
env_data = {param_util.EnvParam(k, v) for (k, v) in envs.items()}
label_data = {
job_model.LabelParam(k, v)
for (k, v) in labels.items()
}
# This is mostly an extraction dsubs argument parsing here:
# https://github.com/googlegenomics/dsub/blob/master/dsub/lib/param_util.py#L720
# Reworked it to handle dictionaries rather than a list of items
# of the form 'key=val'
input_file_param_util = param_util.InputFileParamUtil('input')
input_data = set()
for (recursive, items) in ((False, inputs.items()),
(True, inputs_recursive.items())):
for (name, value) in items:
name = input_file_param_util.get_variable_name(name)
input_data.add(
input_file_param_util.make_param(
name, value, recursive))
output_file_param_util = param_util.OutputFileParamUtil('output')
output_data = set()
for (recursive, items) in ((False, outputs.items()),
(True, outputs_recursive.items())):
for (name, value) in items:
name = output_file_param_util.get_variable_name(name)
output_data.add(
output_file_param_util.make_param(
name, value, recursive))
job_params = {
'envs': env_data,
'inputs': input_data,
'outputs': output_data,
'labels': label_data,
}
if task_count > 1:
task_descriptors = [
job_model.TaskDescriptor({'task-id': i + 1}, {
'envs': env_data,
'inputs': input_data,
'outputs': output_data,
'labels': label_data,
}, job_model.Resources()) for i in xrange(task_count)
]
all_task_data = [{
'task-id': i + 1
} for i in xrange(task_count)]
else:
task_descriptors = [
job_model.TaskDescriptor({'task-id': None}, {
'labels': set(),
'envs': set(),
'inputs': set(),
'outputs': set()
}, job_model.Resources())
]
return execute_redirect_stdout(
lambda: dsub.run(self.provider,
resources,
job_params,
task_descriptors,
name=name,
command=command,
wait=wait,
disable_warning=True))