def test_files_in_command(self, mock_retrieve_call, mock_os_mkdir, mock_isdir): def new_retrieve(arg1): return { 'files1_out': ['/test/file1/foo.txt', '/test/file1/bar.txt'], } mock_retrieve_call.side_effect = new_retrieve job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env() job_interface_dict['command_arguments'] = '${files1}' job_interface_dict['input_data'] = [{ 'name': 'files1', 'type': 'files', 'required': True, }] job_data_dict['input_data'].append({ 'name': 'files1', 'file_ids': [1, 2, 3], }) job_data_dict['output_data'].append({ 'name': 'files1_out', 'workspace_id': self.workspace.id, }) job_interface = JobInterface(job_interface_dict) job_data = JobData(job_data_dict) job_environment = job_environment_dict job_exe_id = 1 job_interface.perform_pre_steps(job_data, job_environment) job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id) expected_command_arguments = os.path.join(SCALE_JOB_EXE_INPUT_PATH, 'files1') self.assertEqual(job_command_arguments, expected_command_arguments, 'expected a different command from pre_steps')
def test_file_in_command(self, mock_retrieve_call, mock_os_mkdir, mock_get_one_file, mock_setup_upload): job_work_dir = "/test" job_exe_id = 1 job_input_dir = get_job_exe_input_data_dir(job_exe_id) job_output_dir = os.path.join(job_work_dir, u'outputs') def new_retrieve(arg1, arg2, arg3): return {u'file1_out': [input_file_path]} input_file_path = os.path.join(job_input_dir, 'file1', 'foo.txt') mock_retrieve_call.side_effect = new_retrieve mock_get_one_file.side_effect = lambda(arg1): input_file_path job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env() job_interface_dict[u'command_arguments'] = u'${file1}' job_interface_dict[u'input_data'] = [{u'name' : u'file1', u'type' : u'file', 'required' : True}] job_data_dict[u'input_data'].append({u'name': u'file1', u'file_id': 1}) job_data_dict[u'output_data'].append({u'name': u'file1_out', u'workspace_id': self.workspace.id}) job_interface = JobInterface(job_interface_dict) job_data = JobData(job_data_dict) job_environment = JobEnvironment(job_environment_dict) job_interface.perform_pre_steps(job_data, job_environment, job_exe_id) job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id) self.assertEqual(job_command_arguments, input_file_path, u'expected a different command from pre_steps') mock_setup_upload.assert_called_once_with(get_job_exe_output_data_dir(job_exe_id), get_job_exe_output_work_dir(job_exe_id), self.workspace)
def test_file_in_command(self, mock_retrieve_call, mock_os_mkdir, mock_get_one_file, mock_isdir): job_exe_id = 1 def new_retrieve(arg1): return { 'file1_out': [input_file_path], } input_file_path = os.path.join(SCALE_JOB_EXE_INPUT_PATH, 'file1', 'foo.txt') mock_retrieve_call.side_effect = new_retrieve mock_get_one_file.side_effect = lambda (arg1): input_file_path job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env() job_interface_dict['command_arguments'] = '${file1}' job_interface_dict['input_data'] = [{ 'name': 'file1', 'type': 'file', 'required': True, }] job_data_dict['input_data'].append({ 'name': 'file1', 'file_id': self.file.id, }) job_data_dict['output_data'].append({ 'name': 'file1_out', 'workspace_id': self.workspace.id, }) job_interface = JobInterface(job_interface_dict) job_data = JobData(job_data_dict) job_environment = job_environment_dict job_interface.perform_pre_steps(job_data, job_environment) job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id) self.assertEqual(job_command_arguments, input_file_path, 'expected a different command from pre_steps')
def test_files_in_command(self, mock_retrieve_call, mock_os_mkdir): def new_retrieve(arg1): return { 'files1_out': ['/test/file1/foo.txt', '/test/file1/bar.txt'], } mock_retrieve_call.side_effect = new_retrieve job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env() job_interface_dict['command_arguments'] = '${files1}' job_interface_dict['input_data'] = [{ 'name': 'files1', 'type': 'files', 'required': True, }] job_data_dict['input_data'].append({ 'name': 'files1', 'file_ids': [1, 2, 3], }) job_data_dict['output_data'].append({ 'name': 'files1_out', 'workspace_id': self.workspace.id, }) job_interface = JobInterface(job_interface_dict) job_data = JobData(job_data_dict) job_environment = JobEnvironment(job_environment_dict) job_exe_id = 1 job_interface.perform_pre_steps(job_data, job_environment) job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id) expected_command_arguments = os.path.join(SCALE_JOB_EXE_INPUT_PATH, 'files1') self.assertEqual(job_command_arguments, expected_command_arguments, 'expected a different command from pre_steps')
def test_file_in_command(self, mock_retrieve_call, mock_os_mkdir, mock_get_one_file): job_exe_id = 1 def new_retrieve(arg1): return { 'file1_out': [input_file_path], } input_file_path = os.path.join(SCALE_JOB_EXE_INPUT_PATH, 'file1', 'foo.txt') mock_retrieve_call.side_effect = new_retrieve mock_get_one_file.side_effect = lambda(arg1): input_file_path job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env() job_interface_dict['command_arguments'] = '${file1}' job_interface_dict['input_data'] = [{ 'name': 'file1', 'type': 'file', 'required': True, }] job_data_dict['input_data'].append({ 'name': 'file1', 'file_id': self.file.id, }) job_data_dict['output_data'].append({ 'name': 'file1_out', 'workspace_id': self.workspace.id, }) job_interface = JobInterface(job_interface_dict) job_data = JobData(job_data_dict) job_environment = JobEnvironment(job_environment_dict) job_interface.perform_pre_steps(job_data, job_environment) job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id) self.assertEqual(job_command_arguments, input_file_path, 'expected a different command from pre_steps')
def test_simple_case(self): job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env() job_interface = JobInterface(job_interface_dict) job_data = JobData(job_data_dict) job_environment = job_environment_dict job_exe_id = 1 job_interface.perform_pre_steps(job_data, job_environment) job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id) self.assertEqual(job_command_arguments, '', 'expected a different command from pre_steps')
def test_simple_case(self): job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env() job_interface = JobInterface(job_interface_dict) job_data = JobData(job_data_dict) job_environment = JobEnvironment(job_environment_dict) job_exe_id = 1 job_interface.perform_pre_steps(job_data, job_environment, 1) job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id) self.assertEqual(job_command_arguments, '', 'expected a different command from pre_steps')
def test_output_dir_in_command(self): job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env() job_interface_dict['command_arguments'] = '${job_output_dir}' job_interface = JobInterface(job_interface_dict) job_data = JobData(job_data_dict) job_environment = job_environment_dict job_exe_id = 1 job_output_dir = SCALE_JOB_EXE_OUTPUT_PATH job_interface.perform_pre_steps(job_data, job_environment) job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id) self.assertEqual(job_command_arguments, job_output_dir, 'expected a different command from pre_steps')
def test_output_dir_in_command(self): job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env() job_interface_dict['command_arguments'] = '${job_output_dir}' job_interface = JobInterface(job_interface_dict) job_data = JobData(job_data_dict) job_environment = JobEnvironment(job_environment_dict) job_exe_id = 1 job_output_dir = file_system.get_job_exe_output_data_dir(job_exe_id) job_interface.perform_pre_steps(job_data, job_environment, 1) job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id) self.assertEqual(job_command_arguments, job_output_dir, 'expected a different command from pre_steps')
def test_complex_command(self): job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env() job_interface_dict[u'command_arguments'] = u'${-f :prop1}' job_interface_dict[u'input_data'] = [{u'name' : u'prop1', u'type' : u'property', 'required' : True}] job_data_dict[u'input_data'].append({u'name': u'prop1', u'value': u'property-value'}) job_interface = JobInterface(job_interface_dict) job_data = JobData(job_data_dict) job_environment = JobEnvironment(job_environment_dict) job_work_dir = "/test" job_exe_id = 1 job_interface.perform_pre_steps(job_data, job_environment, 1) job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id) self.assertEqual(job_command_arguments, u'-f property-value', u'expected a different command from pre_steps')
def test_absent_optional_files_in_command(self): job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env() job_interface_dict['command_arguments'] = '${input_files}' job_interface_dict['input_data'] = [{ 'name': 'input_files', 'type': 'files', 'required': False, }] job_interface = JobInterface(job_interface_dict) job_data = JobData(job_data_dict) job_environment = job_environment_dict job_exe_id = 1 job_interface.perform_pre_steps(job_data, job_environment) job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id) self.assertEqual(job_command_arguments, '', 'expected a different command from pre_steps')
def test_absent_required_files_in_command(self): job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env() job_interface_dict['command_arguments'] = '${input_files}' job_interface_dict['input_data'] = [{ 'name': 'input_files', 'type': 'files', 'required': True, }] job_interface = JobInterface(job_interface_dict) job_data = JobData(job_data_dict) job_environment = job_environment_dict job_exe_id = 1 job_interface.perform_pre_steps(job_data, job_environment) self.assertRaises(InvalidData, lambda: job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id))
def test_file_in_command(self, mock_retrieve_call, mock_os_mkdir, mock_get_one_file, mock_setup_upload): job_exe_id = 1 job_input_dir = file_system.get_job_exe_input_data_dir(job_exe_id) def new_retrieve(arg1, arg2, arg3): return { 'file1_out': [input_file_path], } input_file_path = os.path.join(job_input_dir, 'file1', 'foo.txt') mock_retrieve_call.side_effect = new_retrieve mock_get_one_file.side_effect = lambda (arg1): input_file_path job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env( ) job_interface_dict['command_arguments'] = '${file1}' job_interface_dict['input_data'] = [{ 'name': 'file1', 'type': 'file', 'required': True, }] job_data_dict['input_data'].append({ 'name': 'file1', 'file_id': self.file.id, }) job_data_dict['output_data'].append({ 'name': 'file1_out', 'workspace_id': self.workspace.id, }) job_interface = JobInterface(job_interface_dict) job_data = JobData(job_data_dict) job_environment = JobEnvironment(job_environment_dict) job_interface.perform_pre_steps(job_data, job_environment, job_exe_id) job_command_arguments = job_interface.fully_populate_command_argument( job_data, job_environment, job_exe_id) self.assertEqual(job_command_arguments, input_file_path, 'expected a different command from pre_steps') mock_setup_upload.assert_called_once_with( file_system.get_job_exe_output_data_dir(job_exe_id), file_system.get_job_exe_output_work_dir(job_exe_id), self.workspace)
def test_complex_command(self): job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env() job_interface_dict['command_arguments'] = '${-f :prop1}' job_interface_dict['input_data'] = [{ 'name': 'prop1', 'type': 'property', 'required': True, }] job_data_dict['input_data'].append({ 'name': 'prop1', 'value': 'property-value', }) job_interface = JobInterface(job_interface_dict) job_data = JobData(job_data_dict) job_environment = job_environment_dict job_exe_id = 1 job_interface.perform_pre_steps(job_data, job_environment) job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id) self.assertEqual(job_command_arguments, '-f property-value', 'expected a different command from pre_steps')
def test_property_in_command(self): job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env() job_interface_dict['command_arguments'] = '${prop1}' job_interface_dict['input_data'] = [{ 'name': 'prop1', 'type': 'property', 'required': True, }] job_data_dict['input_data'].append({ 'name': 'prop1', 'value': 'property-value', }) job_interface = JobInterface(job_interface_dict) job_data = JobData(job_data_dict) job_environment = JobEnvironment(job_environment_dict) job_exe_id = 1 job_interface.perform_pre_steps(job_data, job_environment, 1) job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id) self.assertEqual(job_command_arguments, 'property-value', 'expected a different command from pre_steps')