def __init__(self, build_request):
"""
:type build_request: BuildRequest
"""
self._logger = get_logger(__name__)
self._build_id = self._build_id_counter.increment()
self.build_request = build_request
self._artifacts_archive_file = None
self._build_artifact = None
""" :type : BuildArtifact"""
self._error_message = None
self.is_prepared = False
self._preparation_coin = SingleUseCoin(
) # protects against separate threads calling prepare() more than once
self._is_canceled = False
self._project_type = None
self._build_completion_lock = Lock(
) # protects against more than one thread detecting the build's finish
self._slaves_allocated = []
self._num_executors_allocated = 0
self._num_executors_in_use = 0
self._max_executors = float('inf')
self._max_executors_per_slave = float('inf')
self._all_subjobs_by_id = {}
self._unstarted_subjobs = None
self._finished_subjobs = None
self._postbuild_tasks_are_finished = False
self._teardowns_finished = False
self._timing_file_path = None
def setup_build(self, build_id, project_type_params, build_executor_start_index):
"""
Usually called once per build to do build-specific setup. Will block any subjobs from executing until setup
completes. The actual setup is performed on another thread and will unblock subjobs (via an Event) once it
finishes.
:param build_id: The id of the build to run setup on
:type build_id: int
:param project_type_params: The parameters that define the project_type this build will execute in
:type project_type_params: dict
:param build_executor_start_index: How many executors have alreayd been allocated on other slaves for
this build
:type build_executor_start_index: int
"""
self._logger.info('Executing setup for build {} (type: {}).', build_id, project_type_params.get('type'))
self._current_build_id = build_id
self._build_teardown_coin = SingleUseCoin() # protects against build_teardown being executed multiple times
# create an project_type instance for build-level operations
self._project_type = util.create_project_type(project_type_params)
# verify all executors are idle
if not self._idle_executors.full():
raise RuntimeError('Slave tried to setup build but not all executors are idle. ({}/{} executors idle.)'
.format(self._idle_executors.qsize(), self._num_executors))
# Collect all the executors to pass to project_type.fetch_project(). This will create a new project_type for
# each executor (for subjob-level operations).
executors = list(self._idle_executors.queue)
SafeThread(
target=self._async_setup_build,
name='Bld{}-Setup'.format(build_id),
args=(executors, project_type_params, build_executor_start_index)
).start()
def __init__(self, build_request):
"""
:type build_request: BuildRequest
"""
self._logger = get_logger(__name__)
self._build_id = self._build_id_counter.increment()
self._build_request = build_request
self._artifacts_archive_file = None
self._build_artifact = None
self._error_message = None
self._preparation_coin = SingleUseCoin(
) # protects against separate threads calling prepare() more than once
self._project_type = None
self._build_completion_lock = Lock(
) # protects against more than one thread detecting the build's finish
self._all_subjobs_by_id = {}
self._unstarted_subjobs = None # WIP(joey): Move subjob queues to BuildScheduler class.
self._finished_subjobs = None
self._failed_atoms = None
self._postbuild_tasks_are_finished = False # WIP(joey): Remove and use build state.
self._timing_file_path = None
self._state_machine = BuildFsm(build_id=self._build_id,
enter_state_callbacks={
BuildState.ERROR:
self._on_enter_error_state,
BuildState.CANCELED:
self._on_enter_canceled_state,
})
def __init__(self, build_request):
"""
:type build_request: BuildRequest
"""
self._logger = get_logger(__name__)
self._build_id = self._build_id_counter.increment()
self.build_request = build_request
self._artifacts_archive_file = None
self._build_artifact = None
""" :type : BuildArtifact"""
self._error_message = None
self.is_prepared = False
self._setup_is_started = False
self._preparation_coin = SingleUseCoin(
) # protects against separate threads calling prepare() more than once
self._is_canceled = False
self._project_type = None
self._build_completion_lock = Lock(
) # protects against more than one thread detecting the build's finish
self._all_subjobs_by_id = {}
self._unstarted_subjobs = None # WIP: Move subjob queues to BuildScheduler class.
self._finished_subjobs = None
self._failed_atoms = None
self._postbuild_tasks_are_finished = False
self._timing_file_path = None
self._state_timestamps = {status: None
for status in BuildStatus
} # initialize all timestamps to None
self._record_state_timestamp(BuildStatus.QUEUED)
def test_coin_spend_returns_true_only_once(self):
coin = SingleUseCoin()
self.assertTrue(coin.spend(),
'First call to spend() should return True.')
self.assertFalse(coin.spend(),
'Subsequent calls to spend() should return False.')
self.assertFalse(coin.spend(),
'Subsequent calls to spend() should return False.')
def __init__(self, build_request):
"""
:type build_request: BuildRequest
"""
self._logger = get_logger(__name__)
self._build_id = self._build_id_counter.increment()
self._build_request = build_request
self._artifacts_tar_file = None # DEPRECATED - Use zip file instead
self._artifacts_zip_file = None
self._build_artifact = None
self._error_message = None
self._preparation_coin = SingleUseCoin(
) # protects against separate threads calling prepare() more than once
self._project_type = None
self._build_completion_lock = Lock(
) # protects against more than one thread detecting the build's finish
self._all_subjobs_by_id = OrderedDict()
self._unstarted_subjobs = None # WIP(joey): Move subjob queues to BuildScheduler class.
self._finished_subjobs = None
self._failed_atoms = None
self._postbuild_tasks_are_finished = False # WIP(joey): Remove and use build state.
self._timing_file_path = None
leave_state_callbacks = {
build_state: self._on_leave_state
for build_state in BuildState
}
self._state_machine = BuildFsm(
build_id=self._build_id,
enter_state_callbacks={
BuildState.ERROR: self._on_enter_error_state,
BuildState.CANCELED: self._on_enter_canceled_state,
BuildState.PREPARING: self._on_enter_preparing_state,
},
leave_state_callbacks=leave_state_callbacks)
# Number of times build_setup has failed on this build. If
# setup_failures increases beyond MAX_SETUP_FAILURES, the build is
# cancelled
self.setup_failures = 0
def test_signal_shutdown_process_disconnects_from_master_before_killing_executors(self):
disconnect_api_url = 'http://{}/v1/slave/1'.format(self._FAKE_MASTER_URL)
mock_executor = self.patch('app.slave.cluster_slave.SubjobExecutor').return_value
parent_mock = MagicMock() # create a parent mock so we can assert on the order of child mock calls.
parent_mock.attach_mock(self.mock_network, 'mock_network')
parent_mock.attach_mock(mock_executor, 'mock_executor')
slave = self._create_cluster_slave(num_executors=3)
slave.connect_to_master(self._FAKE_MASTER_URL)
slave._build_teardown_coin = SingleUseCoin()
self.trigger_graceful_app_shutdown()
expected_disconnect_call = call.mock_network.put_with_digest(disconnect_api_url, request_params=ANY,
secret=ANY, error_on_failure=ANY)
expected_kill_executor_call = call.mock_executor.kill()
self.assertEqual(1, parent_mock.method_calls.count(expected_disconnect_call),
'Graceful shutdown should cause the slave to make a disconnect call to the master.')
self.assertEqual(3, parent_mock.method_calls.count(expected_kill_executor_call),
'Graceful shutdown should cause the slave to kill all its executors.')
self.assertLess(parent_mock.method_calls.index(expected_disconnect_call),
parent_mock.method_calls.index(expected_kill_executor_call),
'Graceful shutdown should disconnect from the master before killing its executors.')
