def _create_slave_allocator(self, **kwargs):
"""
Create a slave allocator for testing.
:param kwargs: Any constructor parameters for the slave; if none are specified, test defaults will be used.
:rtype: SlaveAllocator
"""
return SlaveAllocator(Mock(spec_set=BuildSchedulerPool))
def __init__(self):
self._logger = get_logger(__name__)
self._master_results_path = Configuration['results_directory']
self._slave_registry = SlaveRegistry.singleton()
self._scheduler_pool = BuildSchedulerPool()
self._build_request_handler = BuildRequestHandler(self._scheduler_pool)
self._build_request_handler.start()
self._slave_allocator = SlaveAllocator(self._scheduler_pool)
self._slave_allocator.start()
# The best practice for determining the number of threads to use is
# the number of threads per core multiplied by the number of physical
# cores. So for example, with 10 cores, 2 sockets and 2 per core, the
# max would be 40.
#
# Currently we use threads for incrementing/decrementing slave executor
# counts (lock acquisition) and tearing down the slave (network IO). 32 threads should be
# plenty for these tasks. In the case of heavy load, the bottle neck will be the number
# of executors, not the time it takes to lock/unlock the executor counts or the number of
# teardown requests. Tweak the number to find the sweet spot if you feel this is the case.
self._thread_pool_executor = ThreadPoolExecutor(max_workers=32)
# Asynchronously delete (but immediately rename) all old builds when master starts.
# Remove this if/when build numbers are unique across master starts/stops
if os.path.exists(self._master_results_path):
fs.async_delete(self._master_results_path)
fs.create_dir(self._master_results_path)
# Configure heartbeat tracking
self._unresponsive_slaves_cleanup_interval = Configuration[
'unresponsive_slaves_cleanup_interval']
self._hb_scheduler = sched.scheduler()
SlavesCollector.register_slaves_metrics_collector(
lambda: self._slave_registry.get_all_slaves_by_id().values())
def _create_slave_allocator(self, **kwargs):
"""
Create a slave allocator for testing.
:param kwargs: Any constructor parameters for the slave; if none are specified, test defaults will be used.
:rtype: SlaveAllocator
"""
kwargs.setdefault('build_request_handler', Mock())
return SlaveAllocator(**kwargs)
def __init__(self):
self._logger = get_logger(__name__)
self._master_results_path = Configuration['results_directory']
self._all_slaves_by_url = {}
self._all_builds_by_id = OrderedDict()
self._build_request_handler = BuildRequestHandler()
self._build_request_handler.start()
self._slave_allocator = SlaveAllocator(self._build_request_handler)
self._slave_allocator.start()
# Asynchronously delete (but immediately rename) all old builds when master starts.
# Remove this if/when build numbers are unique across master starts/stops
if os.path.exists(self._master_results_path):
fs.async_delete(self._master_results_path)
fs.create_dir(self._master_results_path)
def __init__(self):
self._logger = get_logger(__name__)
self._master_results_path = Configuration['results_directory']
self._all_slaves_by_url = {}
self._scheduler_pool = BuildSchedulerPool()
self._build_request_handler = BuildRequestHandler(self._scheduler_pool)
self._build_request_handler.start()
self._slave_allocator = SlaveAllocator(self._scheduler_pool)
self._slave_allocator.start()
# Initialize the database connection before we initialize a BuildStore
Connection.create(Configuration['database_url'])
UnhandledExceptionHandler.singleton().add_teardown_callback(
BuildStore.clean_up)
# The best practice for determining the number of threads to use is
# the number of threads per core multiplied by the number of physical
# cores. So for example, with 10 cores, 2 sockets and 2 per core, the
# max would be 40.
#
# Currently we use threads for incrementing/decrementing slave executor
# counts (lock acquisition) and tearing down the slave (network IO). 32 threads should be
# plenty for these tasks. In the case of heavy load, the bottle neck will be the number
# of executors, not the time it takes to lock/unlock the executor counts or the number of
# teardown requests. Tweak the number to find the sweet spot if you feel this is the case.
self._thread_pool_executor = ThreadPoolExecutor(max_workers=32)
# Asynchronously delete (but immediately rename) all old builds when master starts.
# Remove this if/when build numbers are unique across master starts/stops
# TODO: We can remove this code since we persist builds across master restarts
# if os.path.exists(self._master_results_path):
# fs.async_delete(self._master_results_path)
# fs.create_dir(self._master_results_path)
SlavesCollector.register_slaves_metrics_collector(
lambda: self.all_slaves_by_id().values())
