def test_run_job_config_setup_calls_project_types_run_job_config_setup(self):
executor = SubjobExecutor(1)
executor._project_type = Mock()
executor.run_job_config_setup()
executor._project_type.run_job_config_setup.assert_called_with()
def test_execute_subjob_passes_correct_build_executor_index_to_execute_command_in_project(self):
Configuration['artifact_directory'] = expanduser('~')
executor = SubjobExecutor(1)
executor._project_type = Mock()
executor._project_type.execute_command_in_project = Mock(return_value=(1, 2))
self.patch('app.slave.subjob_executor.fs_util')
self.patch('app.slave.subjob_executor.shutil')
output_file_mock = self.patch('app.slave.subjob_executor.open', new=mock_open(read_data=''), create=True).return_value
os = self.patch('app.slave.subjob_executor.os')
os.path = Mock()
os.path.join = Mock(return_value='path')
atomic_commands = ['command']
executor.id = 2
expected_env_vars = {
'ARTIFACT_DIR': join(expanduser('~'), '1', 'artifact_2_0'),
'ATOM_ID': 0,
'EXECUTOR_INDEX': 2,
'MACHINE_EXECUTOR_INDEX': 2,
'BUILD_EXECUTOR_INDEX': 8
}
executor.execute_subjob(build_id=1, subjob_id=2, atomic_commands=atomic_commands,
base_executor_index=6)
executor._project_type.execute_command_in_project.assert_called_with('command', expected_env_vars,
output_file=output_file_mock)
def test_run_job_config_setup_calls_project_types_run_job_config_setup(
self):
executor = SubjobExecutor(1)
executor._project_type = Mock()
executor.run_job_config_setup()
executor._project_type.run_job_config_setup.assert_called_with()
def test_configure_project_type_with_existing_project_type_calls_teardown(self):
executor = SubjobExecutor(1)
executor._project_type = Mock()
self.patch('app.slave.subjob_executor.util')
executor.configure_project_type({})
executor._project_type.teardown_executor.assert_called_once()
def test_configure_project_type_with_existing_project_type_calls_teardown(
self):
executor = SubjobExecutor(1)
executor._project_type = Mock()
self.patch('app.slave.subjob_executor.util')
executor.configure_project_type({})
executor._project_type.teardown_executor.assert_called_once()
def test_configure_project_type_passes_project_type_params_and_calls_setup_executor(self):
project_type_params = {'test': 'value'}
util = self.patch('app.slave.subjob_executor.util')
util.create_project_type = Mock(return_value=Mock())
executor = SubjobExecutor(1)
executor.configure_project_type(project_type_params)
util.create_project_type.assert_called_with(project_type_params)
executor._project_type.setup_executor.assert_called_with()
def test_configure_project_type_passes_project_type_params_and_calls_setup_executor(
self):
project_type_params = {'test': 'value'}
util = self.patch('app.slave.subjob_executor.util')
util.create_project_type = Mock(return_value=Mock())
executor = SubjobExecutor(1)
executor.configure_project_type(project_type_params)
util.create_project_type.assert_called_with(project_type_params)
executor._project_type.setup_executor.assert_called_with()
def __init__(self, port, host, num_executors=10):
"""
:param port: The port number the slave service is running on
:type port: int
:param host: The hostname at which the slave is reachable
:type host: str
:param num_executors: The number of executors this slave should operate with -- this determines how many
concurrent subjobs the slave can execute.
:type num_executors: int
"""
self.port = port
self.host = host
self.is_alive = True
self._slave_id = None
self._num_executors = num_executors
self._logger = log.get_logger(__name__)
self._idle_executors = Queue(maxsize=num_executors)
self.executors_by_id = {}
for executor_id in range(num_executors):
executor = SubjobExecutor(executor_id)
self._idle_executors.put(executor)
self.executors_by_id[executor_id] = executor
self._master_url = None
self._network = Network(min_connection_poolsize=num_executors)
self._master_api = None # wait until we connect to a master first
self._project_type = None # this will be instantiated during build setup
self._current_build_id = None
self._build_teardown_coin = None
def test_execute_subjob_passes_correct_build_executor_index_to_execute_command_in_project(
self):
Configuration['artifact_directory'] = expanduser('~')
executor = SubjobExecutor(1)
executor._project_type = Mock()
executor._project_type.execute_command_in_project = Mock(
return_value=(1, 2))
self.patch('app.slave.subjob_executor.fs_util')
self.patch('app.slave.subjob_executor.shutil')
output_file_mock = self.patch('app.slave.subjob_executor.open',
new=mock_open(read_data=''),
create=True).return_value
os = self.patch('app.slave.subjob_executor.os')
os.path = Mock()
os.path.join = Mock(return_value='path')
atomic_commands = ['command']
executor.id = 2
expected_env_vars = {
'ARTIFACT_DIR': join(expanduser('~'), '1', 'artifact_2_0'),
'ATOM_ID': 0,
'EXECUTOR_INDEX': 2,
'MACHINE_EXECUTOR_INDEX': 2,
'BUILD_EXECUTOR_INDEX': 8
}
executor.execute_subjob(build_id=1,
subjob_id=2,
atomic_commands=atomic_commands,
base_executor_index=6)
executor._project_type.execute_command_in_project.assert_called_with(
'command', expected_env_vars, output_file=output_file_mock)
def test_execute_subjob_passes_correct_build_executor_index_to_execute_command_in_project(
self):
executor = SubjobExecutor(1)
executor._project_type = Mock()
executor._project_type.execute_command_in_project = Mock(
return_value=(1, 2))
self.patch('app.slave.subjob_executor.fs_util')
self.patch('app.slave.subjob_executor.shutil')
os = self.patch('app.slave.subjob_executor.os')
os.path = Mock()
os.path.join = Mock(return_value='path')
atomic_commands = ['command']
executor.id = 2
expected_env_vars = {
'ARTIFACT_DIR': 'path',
'ATOM_ID': 0,
'EXECUTOR_INDEX': 2,
'MACHINE_EXECUTOR_INDEX': 2,
'BUILD_EXECUTOR_INDEX': 8
}
executor.execute_subjob(build_id=1,
subjob_id=2,
subjob_artifact_dir='dir',
atomic_commands=atomic_commands,
base_executor_index=6)
executor._project_type.execute_command_in_project.assert_called_with(
'command', expected_env_vars)
def test_execute_subjob_passes_correct_build_executor_index_to_execute_command_in_project(self):
executor = SubjobExecutor(1)
executor._project_type = Mock()
executor._project_type.execute_command_in_project = Mock(return_value=(1, 2))
self.patch('app.slave.subjob_executor.fs_util')
self.patch('app.slave.subjob_executor.shutil')
os = self.patch('app.slave.subjob_executor.os')
os.path = Mock()
os.path.join = Mock(return_value='path')
atomic_commands = ['command']
executor.id = 2
expected_env_vars = {
'ARTIFACT_DIR': 'path',
'ATOM_ID': 0,
'EXECUTOR_INDEX': 2,
'MACHINE_EXECUTOR_INDEX': 2,
'BUILD_EXECUTOR_INDEX': 8
}
executor.execute_subjob(build_id=1, subjob_id=2, subjob_artifact_dir='dir', atomic_commands=atomic_commands,
base_executor_index=6)
executor._project_type.execute_command_in_project.assert_called_with('command', expected_env_vars)