def test_foo(self, retries, max_preemptible_attempts, list_of_list_of_operations, expected): def chronology(list_of_list_of_operations): for operations in list_of_list_of_operations: self.provider.set_operations(operations) yield 1 establish_chronology(chronology(list_of_list_of_operations)) job_descriptor = job_model.JobDescriptor( job_metadata={'create-time': 123456}, job_params=None, job_resources=job_model.Resources( logging=job_model.LoggingParam('gs://buck/logs', job_model.P_GCS), max_preemptible_attempts=param_util.PreemptibleParam( max_preemptible_attempts)), task_descriptors=[ job_model.TaskDescriptor( task_metadata={ 'task-id': 123, 'create-time': 123456 }, task_params=None, task_resources=None) ]) poll_interval = 1 ret = dsub_command._wait_and_retry( self.provider, 'job-1', poll_interval, retries, job_descriptor, summary=False) tasks = self.provider.lookup_job_tasks({'*'}) # First, the number of tasks returned by lookup_job_tasks should be equal # to the total number of task attempts. expected_num_of_tasks = len(list_of_list_of_operations[-1]) self.assertEqual(len(tasks), expected_num_of_tasks) # Second, check that the return value of _wait_and_retry is correct. self.assertEqual(ret, expected)
def _make_task(job_create_time, task_create_time, task_id): return local.LocalTask(job_descriptor=job_model.JobDescriptor( job_metadata={'create-time': job_create_time}, job_params=None, job_resources=None, task_descriptors=[ job_model.TaskDescriptor(task_metadata={ 'task-id': task_id, 'create-time': task_create_time }, task_params=None, task_resources=None) ]), task_status=None, log_detail=None, end_time=None, last_update=None, pid=None, events=None)
def dsub_start_job(command, job_name=None, envs=None, labels=None, inputs=None, inputs_recursive=None, outputs=None, outputs_recursive=None, wait=False): envs = envs or {} labels = labels or {} inputs = inputs or {} inputs_recursive = inputs_recursive or {} outputs = outputs or {} outputs_recursive = outputs_recursive or {} labels['test-token'] = test_setup.TEST_TOKEN labels['test-name'] = test_setup.TEST_NAME logging = param_util.build_logging_param(test.LOGGING) job_resources = job_model.Resources(image='ubuntu', logging=logging, zones=['us-central1-*']) env_data = {job_model.EnvParam(k, v) for (k, v) in envs.items()} label_data = {job_model.LabelParam(k, v) for (k, v) in labels.items()} input_file_param_util = param_util.InputFileParamUtil('input') input_data = set() for (recursive, items) in ((False, inputs.items()), (True, inputs_recursive.items())): for (name, value) in items: name = input_file_param_util.get_variable_name(name) input_data.add( input_file_param_util.make_param(name, value, recursive)) output_file_param_util = param_util.OutputFileParamUtil('output') output_data = set() for (recursive, items) in ((False, outputs.items()), (True, outputs_recursive.items())): for (name, value) in items: name = output_file_param_util.get_variable_name(name) output_data.add( output_file_param_util.make_param(name, value, recursive)) job_params = { 'envs': env_data, 'inputs': input_data, 'outputs': output_data, 'labels': label_data, } task_descriptors = [ job_model.TaskDescriptor({'task-id': None}, { 'envs': set(), 'labels': set(), 'inputs': set(), 'outputs': set(), }, job_model.Resources()) ] return dsub.run(get_dsub_provider(), job_resources, job_params, task_descriptors, name=job_name, command=command, wait=wait, disable_warning=True)
job_model.EnvParam('VAR3', 'VAL3'), job_model.EnvParam('VAR4', 'VAL4'), job_model.EnvParam('VAR5', 'VAL5'), }, 'labels': set(), 'inputs': set(), 'outputs': set(), 'mounts': set(), }, task_descriptors=[ job_model.TaskDescriptor( task_metadata={'task-id': None}, task_resources=job_model.Resources( logging_path= 'gs://b/dsub/sh/local/env_list/env_list/logging/env_list.log'), task_params={ 'envs': set(), 'labels': set(), 'inputs': set(), 'outputs': set(), }) ]) # The following test data is from the e2e_io_tasks.sh test. # This job uses a --tasks file. # # dsub \ # --logging gs://b/dsub/sh/local/env_list/env_list/logging/env_list.log # --image "ubuntu" \ # --script "script_io_test.sh" \
('glog', 'gs://bucket/my.log', 'gs://bucket/my.log', job_model.P_GCS), ]) def test_logging_param_maker(self, unused_name, uri, expected_out, provider): del unused_name param = param_util.build_logging_param( uri, util_class=param_util.OutputFileParamUtil) self.assertEqual(param.uri, expected_out) self.assertEqual(param.file_provider, provider) TASK_DESCRIPTORS = [ job_model.TaskDescriptor( None, { 'inputs': [ job_model.FileParam( 'IN', uri='gs://in/*', file_provider=job_model.P_GCS) ], 'outputs': [] }, None), job_model.TaskDescriptor( None, { 'inputs': [], 'outputs': [ job_model.FileParam( 'OUT', uri='gs://out/*', file_provider=job_model.P_GCS) ] }, None), ] class TestSubmitValidator(unittest.TestCase):
def start_job(self, command, name=None, envs={}, labels={}, inputs={}, inputs_recursive={}, outputs={}, outputs_recursive={}, task_count=1, wait=False): logging = param_util.build_logging_param(self.log_path) resources = job_model.Resources(image=DOCKER_IMAGE, logging=logging, zones=['us-central1*']) env_data = {param_util.EnvParam(k, v) for (k, v) in envs.items()} label_data = { job_model.LabelParam(k, v) for (k, v) in labels.items() } # This is mostly an extraction dsubs argument parsing here: # https://github.com/googlegenomics/dsub/blob/master/dsub/lib/param_util.py#L720 # Reworked it to handle dictionaries rather than a list of items # of the form 'key=val' input_file_param_util = param_util.InputFileParamUtil('input') input_data = set() for (recursive, items) in ((False, inputs.items()), (True, inputs_recursive.items())): for (name, value) in items: name = input_file_param_util.get_variable_name(name) input_data.add( input_file_param_util.make_param( name, value, recursive)) output_file_param_util = param_util.OutputFileParamUtil('output') output_data = set() for (recursive, items) in ((False, outputs.items()), (True, outputs_recursive.items())): for (name, value) in items: name = output_file_param_util.get_variable_name(name) output_data.add( output_file_param_util.make_param( name, value, recursive)) job_params = { 'envs': env_data, 'inputs': input_data, 'outputs': output_data, 'labels': label_data, } if task_count > 1: task_descriptors = [ job_model.TaskDescriptor({'task-id': i + 1}, { 'envs': env_data, 'inputs': input_data, 'outputs': output_data, 'labels': label_data, }, job_model.Resources()) for i in xrange(task_count) ] all_task_data = [{ 'task-id': i + 1 } for i in xrange(task_count)] else: task_descriptors = [ job_model.TaskDescriptor({'task-id': None}, { 'labels': set(), 'envs': set(), 'inputs': set(), 'outputs': set() }, job_model.Resources()) ] return execute_redirect_stdout( lambda: dsub.run(self.provider, resources, job_params, task_descriptors, name=name, command=command, wait=wait, disable_warning=True))