def __init__(self, port, host, num_executors=10):
"""
:param port: The port number the slave service is running on
:type port: int
:param host: The hostname at which the slave is reachable
:type host: str
:param num_executors: The number of executors this slave should operate with -- this determines how many
concurrent subjobs the slave can execute.
:type num_executors: int
"""
self.port = port
self.host = host
self.is_alive = True
self._slave_id = None
self._num_executors = num_executors
self._logger = log.get_logger(__name__)
self._idle_executors = Queue(maxsize=num_executors)
self.executors_by_id = {}
for executor_id in range(num_executors):
executor = SubjobExecutor(executor_id)
self._idle_executors.put(executor)
self.executors_by_id[executor_id] = executor
self._master_url = None
self._network = Network(min_connection_poolsize=num_executors)
self._master_api = None # wait until we connect to a master first
self._project_type = None # this will be instantiated during build setup
self._current_build_id = None
self._build_teardown_coin = None
def _validate_successful_deployment(self, master_service_url, slaves_to_validate):
"""
Poll the master's /slaves endpoint until either timeout or until all of the slaves have registered with
the master.
Throws exception upon timeout or API response error.
:param master_service_url: the hostname:port for the running master service
:type master_service_url: str
:param slaves_to_validate: the list of slave hostnames (no ports) to deploy to
:type slaves_to_validate: list[str]
"""
master_api = UrlBuilder(master_service_url, BuildRunner.API_VERSION)
slave_api_url = master_api.url('slave')
network = Network()
def all_slaves_registered():
return len(self._registered_slave_hostnames(slave_api_url, network)) == len(slaves_to_validate)
if not wait_for(
boolean_predicate=all_slaves_registered,
timeout_seconds=self._SLAVE_REGISTRY_TIMEOUT_SEC,
poll_period=1,
exceptions_to_swallow=(requests.RequestException, requests.ConnectionError)
):
try:
registered_slaves = self._registered_slave_hostnames(slave_api_url, network)
non_registered_slaves = self._non_registered_slaves(registered_slaves, slaves_to_validate)
except ConnectionError:
self._logger.error('Error contacting {} on the master.'.format(slave_api_url))
raise SystemExit(1)
self._logger.error('Slave registration timed out after {} sec, with slaves {} missing.'.format(
self._SLAVE_REGISTRY_TIMEOUT_SEC, ','.join(non_registered_slaves)))
raise SystemExit(1)
def __init__(self, base_api_url):
"""
:param base_api_url: The base API url of the service (e.g., 'http://localhost:43000')
:type base_api_url: str
"""
self._api = UrlBuilder(base_api_url)
self._network = Network()
self._logger = log.get_logger(__name__)
def setUp(self):
# Configure logging to go to stdout. This makes debugging easier by allowing us to see logs for failed tests.
log.configure_logging('DEBUG')
Secret.set('testsecret')
self.cluster = FunctionalTestCluster(
verbose=self._get_test_verbosity())
self._network = Network()
def test_reset_session_closes_and_recreates_session(self):
first_session = Mock(Session)
second_session = Mock(Session)
self.mock_session_cls.side_effect = [first_session, second_session]
network = Network()
network.reset_session()
self.assertEqual(self.mock_session_cls.call_count, 2,
'Two sessions should be created.')
self.assertEqual(first_session.close.call_count, 1,
'First session should be closed.')
self.assertEqual(second_session.close.call_count, 0,
'Second session should not be closed.')
def __init__(self, slave_url, num_executors):
"""
:type slave_url: str
:type num_executors: int
"""
self.url = slave_url
self.num_executors = num_executors
self.id = self._slave_id_counter.increment()
self._num_executors_in_use = Counter()
self._network = Network(min_connection_poolsize=num_executors)
self.current_build_id = None
self._is_alive = True
self._slave_api = UrlBuilder(slave_url, self.API_VERSION)
self._logger = log.get_logger(__name__)
def __init__(self, master_url, request_params, secret):
"""
:param master_url: The url of the master which the build will be executed on
:type master_url: str
:param request_params: A dict of request params that will be json-encoded and sent in the build request
:type request_params: dict
:type secret: str
"""
self._master_url = self._ensure_url_has_scheme(master_url)
self._request_params = request_params
self._secret = secret
self._build_id = None
self._network = Network()
self._logger = get_logger(__name__)
self._last_build_status_details = None
self._master_api = UrlBuilder(master_url, self.API_VERSION)
self._cluster_master_api_client = ClusterMasterAPIClient(master_url)
def __init__(self, slave_url, num_executors, slave_session_id=None):
"""
:type slave_url: str
:type num_executors: int
:type slave_session_id: str
"""
self.url = slave_url
self.num_executors = num_executors
self.id = self._slave_id_counter.increment()
self._num_executors_in_use = Counter()
self._network = Network(min_connection_poolsize=num_executors)
self.current_build_id = None
self._last_heartbeat_time = datetime.now()
self._is_alive = True
self._is_in_shutdown_mode = False
self._slave_api = UrlBuilder(slave_url, self.API_VERSION)
self._session_id = slave_session_id
self._logger = log.get_logger(__name__)
def is_up(self, service_url, timeout=0.1):
"""
Checks if the service is up
:type service_url: string
:type timeout: float
:rtype: bool
"""
network = Network()
timeout_time = time.time() + timeout
while True:
try:
resp = network.get('http://{}'.format(service_url), timeout=timeout)
if resp and resp.ok:
return True
except (requests.RequestException, ConnectionError):
pass
if time.time() > timeout_time:
break
time.sleep(0.5)
return False
