def test_teardown_called_with_incorrect_build_id_will_raise(self, slave_current_build_id):
slave = ClusterSlave(port=15140, host='uncle.pennybags.gov')
slave._current_build_id = slave_current_build_id
incorrect_build_id = 300
with self.assertRaises(BadRequestError, msg='Teardown should raise error if incorrect build_id specified.'):
slave.teardown_build(incorrect_build_id)
def test_start_working_on_subjob_called_with_incorrect_build_id_will_raise(self, slave_current_build_id):
slave = ClusterSlave(port=15140, host='uncle.pennybags.gov')
slave._current_build_id = slave_current_build_id
incorrect_build_id = 300
with self.assertRaises(BadRequestError, msg='Start subjob should raise error if incorrect build_id specified.'):
slave.start_working_on_subjob(incorrect_build_id, 1, '~/test', ['ls'])
def _create_cluster_slave(self, **kwargs):
"""
Create a ClusterSlave for testing.
:param kwargs: Any constructor parameters for the slave; if none are specified, test defaults will be used.
:rtype: ClusterSlave
"""
kwargs.setdefault('host', self._FAKE_SLAVE_HOST)
kwargs.setdefault('port', self._FAKE_SLAVE_PORT)
return ClusterSlave(**kwargs)
def test_disconnect_request_sent_if_and_only_if_master_is_responsive(self, is_master_responsive):
master_url = 'uncle.pennybags.gov:15139'
slave_connect_api_url = 'http://uncle.pennybags.gov:15139/v1/slave'
slave_disconnect_api_url = 'http://uncle.pennybags.gov:15139/v1/slave/1/disconnect'
if not is_master_responsive:
self.mock_network.get.side_effect = requests.ConnectionError # trigger an exception on get
slave = ClusterSlave(port=15140, host='uncle.pennybags.gov')
slave.connect_to_master(master_url)
slave._async_teardown_build(should_disconnect_from_master=True)
# always expect a connect call, and if the master is responsive also expect a disconnect call
expected_network_post_calls = [call(slave_connect_api_url, ANY)]
if is_master_responsive:
expected_network_post_calls.append(call(slave_disconnect_api_url))
self.mock_network.post.assert_has_calls(expected_network_post_calls, any_order=True)
self.assertEqual(self.mock_network.post.call_count, len(expected_network_post_calls),
'All POST requests should be accounted for in the test.')
def async_run(self, port, master_url, num_executors, log_level,
eventlog_file):
"""
Run a ClusterRunner slave service.
:param port: the port on which to run the slave service
:type port: int | None
:param master_url: the url of the master to which this slave should attach
:type master_url: string | None
:param num_executors: the number of executors the slave service should use
:type num_executors: int | None
:param log_level: the log level at which to do application logging (or None for default log level)
:type log_level: str | None
:param eventlog_file: an optional alternate file in which to write event logs
:type eventlog_file: str | None
"""
num_executors = num_executors or Configuration['num_executors']
master_url = master_url or '{}:{}'.format(
Configuration['master_hostname'], Configuration['master_port'])
port = port or Configuration['port']
log_level = log_level or Configuration['log_level']
eventlog_file = eventlog_file or Configuration['eventlog_file']
log.configure_logging(log_level=log_level,
log_file=Configuration['log_file'].format(port))
analytics.initialize(eventlog_file)
analytics.record_event(analytics.SERVICE_STARTED, service='slave')
cluster_slave = ClusterSlave(
port=port,
num_executors=num_executors,
host=Configuration['hostname'],
)
application = ClusterSlaveApplication(cluster_slave)
ioloop = self._start_application(application, port)
self._write_pid_file(Configuration['slave_pid_file'])
# connect to master once tornado ioloop is running
connect_slave_to_master = functools.partial(
cluster_slave.connect_to_master, master_url=master_url)
ioloop.add_callback(connect_slave_to_master)
# start sending heartbeat after connecting to master
start_slave_heartbeat = functools.partial(
cluster_slave.start_heartbeat_thread)
ioloop.add_callback(start_slave_heartbeat)
ioloop.start() # this call blocks until the server is stopped
ioloop.close(
all_fds=True
) # all_fds=True is necessary here to make sure connections don't hang
self._logger.notice('Slave server was stopped.')
def _is_slave_responsive(self, slave: ClusterSlave) -> bool:
time_since_last_heartbeat = (datetime.now() -
slave.get_last_heartbeat_time()).seconds
return time_since_last_heartbeat < self._unresponsive_slaves_cleanup_interval