def test_foo(self, retries, max_preemptible_attempts,
list_of_list_of_operations, expected):
def chronology(list_of_list_of_operations):
for operations in list_of_list_of_operations:
self.provider.set_operations(operations)
yield 1
establish_chronology(chronology(list_of_list_of_operations))
job_descriptor = job_model.JobDescriptor(
job_metadata={'create-time': 123456},
job_params=None,
job_resources=job_model.Resources(
logging=job_model.LoggingParam('gs://buck/logs', job_model.P_GCS),
max_preemptible_attempts=param_util.PreemptibleParam(
max_preemptible_attempts)),
task_descriptors=[
job_model.TaskDescriptor(
task_metadata={
'task-id': 123,
'create-time': 123456
},
task_params=None,
task_resources=None)
])
poll_interval = 1
ret = dsub_command._wait_and_retry(
self.provider,
'job-1',
poll_interval,
retries,
job_descriptor,
summary=False)
tasks = self.provider.lookup_job_tasks({'*'})
# First, the number of tasks returned by lookup_job_tasks should be equal
# to the total number of task attempts.
expected_num_of_tasks = len(list_of_list_of_operations[-1])
self.assertEqual(len(tasks), expected_num_of_tasks)
# Second, check that the return value of _wait_and_retry is correct.
self.assertEqual(ret, expected)
def _make_task(job_create_time, task_create_time, task_id):
return local.LocalTask(job_descriptor=job_model.JobDescriptor(
job_metadata={'create-time': job_create_time},
job_params=None,
job_resources=None,
task_descriptors=[
job_model.TaskDescriptor(task_metadata={
'task-id': task_id,
'create-time': task_create_time
},
task_params=None,
task_resources=None)
]),
task_status=None,
log_detail=None,
end_time=None,
last_update=None,
pid=None,
events=None)
def dsub_start_job(command,
job_name=None,
envs=None,
labels=None,
inputs=None,
inputs_recursive=None,
outputs=None,
outputs_recursive=None,
wait=False):
envs = envs or {}
labels = labels or {}
inputs = inputs or {}
inputs_recursive = inputs_recursive or {}
outputs = outputs or {}
outputs_recursive = outputs_recursive or {}
labels['test-token'] = test_setup.TEST_TOKEN
labels['test-name'] = test_setup.TEST_NAME
logging = param_util.build_logging_param(test.LOGGING)
job_resources = job_model.Resources(image='ubuntu',
logging=logging,
zones=['us-central1-*'])
env_data = {job_model.EnvParam(k, v) for (k, v) in envs.items()}
label_data = {job_model.LabelParam(k, v) for (k, v) in labels.items()}
input_file_param_util = param_util.InputFileParamUtil('input')
input_data = set()
for (recursive, items) in ((False, inputs.items()),
(True, inputs_recursive.items())):
for (name, value) in items:
name = input_file_param_util.get_variable_name(name)
input_data.add(
input_file_param_util.make_param(name, value, recursive))
output_file_param_util = param_util.OutputFileParamUtil('output')
output_data = set()
for (recursive, items) in ((False, outputs.items()),
(True, outputs_recursive.items())):
for (name, value) in items:
name = output_file_param_util.get_variable_name(name)
output_data.add(
output_file_param_util.make_param(name, value, recursive))
job_params = {
'envs': env_data,
'inputs': input_data,
'outputs': output_data,
'labels': label_data,
}
task_descriptors = [
job_model.TaskDescriptor({'task-id': None}, {
'envs': set(),
'labels': set(),
'inputs': set(),
'outputs': set(),
}, job_model.Resources())
]
return dsub.run(get_dsub_provider(),
job_resources,
job_params,
task_descriptors,
name=job_name,
command=command,
wait=wait,
disable_warning=True)
job_model.EnvParam('VAR3', 'VAL3'),
job_model.EnvParam('VAR4', 'VAL4'),
job_model.EnvParam('VAR5', 'VAL5'),
},
'labels': set(),
'inputs': set(),
'outputs': set(),
'mounts': set(),
},
task_descriptors=[
job_model.TaskDescriptor(
task_metadata={'task-id': None},
task_resources=job_model.Resources(
logging_path=
'gs://b/dsub/sh/local/env_list/env_list/logging/env_list.log'),
task_params={
'envs': set(),
'labels': set(),
'inputs': set(),
'outputs': set(),
})
])
# The following test data is from the e2e_io_tasks.sh test.
# This job uses a --tasks file.
#
# dsub \
# --logging gs://b/dsub/sh/local/env_list/env_list/logging/env_list.log
# --image "ubuntu" \
# --script "script_io_test.sh" \
('glog', 'gs://bucket/my.log', 'gs://bucket/my.log', job_model.P_GCS),
])
def test_logging_param_maker(self, unused_name, uri, expected_out,
provider):
del unused_name
param = param_util.build_logging_param(
uri, util_class=param_util.OutputFileParamUtil)
self.assertEqual(param.uri, expected_out)
self.assertEqual(param.file_provider, provider)
TASK_DESCRIPTORS = [
job_model.TaskDescriptor(
None, {
'inputs': [
job_model.FileParam(
'IN', uri='gs://in/*', file_provider=job_model.P_GCS)
],
'outputs': []
}, None),
job_model.TaskDescriptor(
None, {
'inputs': [],
'outputs': [
job_model.FileParam(
'OUT', uri='gs://out/*', file_provider=job_model.P_GCS)
]
}, None),
]
class TestSubmitValidator(unittest.TestCase):
def start_job(self,
command,
name=None,
envs={},
labels={},
inputs={},
inputs_recursive={},
outputs={},
outputs_recursive={},
task_count=1,
wait=False):
logging = param_util.build_logging_param(self.log_path)
resources = job_model.Resources(image=DOCKER_IMAGE,
logging=logging,
zones=['us-central1*'])
env_data = {param_util.EnvParam(k, v) for (k, v) in envs.items()}
label_data = {
job_model.LabelParam(k, v)
for (k, v) in labels.items()
}
# This is mostly an extraction dsubs argument parsing here:
# https://github.com/googlegenomics/dsub/blob/master/dsub/lib/param_util.py#L720
# Reworked it to handle dictionaries rather than a list of items
# of the form 'key=val'
input_file_param_util = param_util.InputFileParamUtil('input')
input_data = set()
for (recursive, items) in ((False, inputs.items()),
(True, inputs_recursive.items())):
for (name, value) in items:
name = input_file_param_util.get_variable_name(name)
input_data.add(
input_file_param_util.make_param(
name, value, recursive))
output_file_param_util = param_util.OutputFileParamUtil('output')
output_data = set()
for (recursive, items) in ((False, outputs.items()),
(True, outputs_recursive.items())):
for (name, value) in items:
name = output_file_param_util.get_variable_name(name)
output_data.add(
output_file_param_util.make_param(
name, value, recursive))
job_params = {
'envs': env_data,
'inputs': input_data,
'outputs': output_data,
'labels': label_data,
}
if task_count > 1:
task_descriptors = [
job_model.TaskDescriptor({'task-id': i + 1}, {
'envs': env_data,
'inputs': input_data,
'outputs': output_data,
'labels': label_data,
}, job_model.Resources()) for i in xrange(task_count)
]
all_task_data = [{
'task-id': i + 1
} for i in xrange(task_count)]
else:
task_descriptors = [
job_model.TaskDescriptor({'task-id': None}, {
'labels': set(),
'envs': set(),
'inputs': set(),
'outputs': set()
}, job_model.Resources())
]
return execute_redirect_stdout(
lambda: dsub.run(self.provider,
resources,
job_params,
task_descriptors,
name=name,
command=command,
wait=wait,
disable_warning=True))