def __init__(self):
self._logger = get_logger(__name__)
self._master_results_path = Configuration['results_directory']
self._slave_registry = SlaveRegistry.singleton()
self._scheduler_pool = BuildSchedulerPool()
self._build_request_handler = BuildRequestHandler(self._scheduler_pool)
self._build_request_handler.start()
self._slave_allocator = SlaveAllocator(self._scheduler_pool)
self._slave_allocator.start()
# The best practice for determining the number of threads to use is
# the number of threads per core multiplied by the number of physical
# cores. So for example, with 10 cores, 2 sockets and 2 per core, the
# max would be 40.
#
# Currently we use threads for incrementing/decrementing slave executor
# counts (lock acquisition) and tearing down the slave (network IO). 32 threads should be
# plenty for these tasks. In the case of heavy load, the bottle neck will be the number
# of executors, not the time it takes to lock/unlock the executor counts or the number of
# teardown requests. Tweak the number to find the sweet spot if you feel this is the case.
self._thread_pool_executor = ThreadPoolExecutor(max_workers=32)
# Asynchronously delete (but immediately rename) all old builds when master starts.
# Remove this if/when build numbers are unique across master starts/stops
if os.path.exists(self._master_results_path):
fs.async_delete(self._master_results_path)
fs.create_dir(self._master_results_path)
# Configure heartbeat tracking
self._unresponsive_slaves_cleanup_interval = Configuration[
'unresponsive_slaves_cleanup_interval']
self._hb_scheduler = sched.scheduler()
SlavesCollector.register_slaves_metrics_collector(
lambda: self._slave_registry.get_all_slaves_by_id().values())
def __init__(self):
self._logger = get_logger(__name__)
self._all_slaves_by_url = {}
self._all_builds_by_id = OrderedDict() # This is an OrderedDict so we can more easily implement get_queue()
self._builds_waiting_for_slaves = Queue()
self._request_queue = Queue()
self._request_handler = SerialRequestHandler()
self._request_queue_worker_thread = SafeThread(
target=self._build_preparation_loop, name='RequestHandlerLoop', daemon=True)
self._request_queue_worker_thread.start()
self._slave_allocation_worker_thread = SafeThread(
target=self._slave_allocation_loop, name='SlaveAllocationLoop', daemon=True)
self._slave_allocation_worker_thread.start()
self._master_results_path = Configuration['results_directory']
# It's important that idle slaves are only in the queue once so we use OrderedSet
self._idle_slaves = OrderedSetQueue()
# Asynchronously delete (but immediately rename) all old builds when master starts.
# Remove this if/when build numbers are unique across master starts/stops
if os.path.exists(self._master_results_path):
fs.async_delete(self._master_results_path)
fs.create_dir(self._master_results_path)
def test_async_delete_calls_correct_commands(self):
popen_mock = self.patch('subprocess.Popen')
move_mock = self.patch('shutil.move')
self.patch('os.path.isdir').return_value = True
mkdtemp_mock = self.patch('tempfile.mkdtemp')
mkdtemp_mock.return_value = '/tmp/dir'
fs.async_delete('/some/dir')
move_mock.assert_called_with('/some/dir', '/tmp/dir')
popen_mock.assert_called_with(['rm', '-rf', '/tmp/dir'])
def test_async_delete_calls_correct_commands(self):
popen_mock = self.patch("app.util.fs.Popen_with_delayed_expansion")
move_mock = self.patch("shutil.move")
self.patch("os.path.isdir").return_value = True
mkdtemp_mock = self.patch("tempfile.mkdtemp")
mkdtemp_mock.return_value = "/tmp/dir"
fs.async_delete("/some/dir")
move_mock.assert_called_with("/some/dir", "/tmp/dir")
popen_mock.assert_called_with(["rm", "-rf", "/tmp/dir"])
def test_async_delete_calls_correct_commands(self):
popen_mock = self.patch('subprocess.Popen')
move_mock = self.patch('shutil.move')
self.patch('os.path.isdir').return_value = True
mkdtemp_mock = self.patch('tempfile.mkdtemp')
mkdtemp_mock.return_value = '/tmp/dir'
fs.async_delete('/some/dir')
move_mock.assert_called_with('/some/dir', '/tmp/dir')
popen_mock.assert_called_with(['rm', '-rf', '/tmp/dir'])
def __init__(self):
self._logger = get_logger(__name__)
self._master_results_path = Configuration['results_directory']
self._all_slaves_by_url = {}
self._all_builds_by_id = OrderedDict()
self._build_request_handler = BuildRequestHandler()
self._build_request_handler.start()
self._slave_allocator = SlaveAllocator(self._build_request_handler)
self._slave_allocator.start()
# Asynchronously delete (but immediately rename) all old builds when master starts.
# Remove this if/when build numbers are unique across master starts/stops
if os.path.exists(self._master_results_path):
fs.async_delete(self._master_results_path)
fs.create_dir(self._master_results_path)
def __init__(self):
self._logger = get_logger(__name__)
self._master_results_path = Configuration["results_directory"]
self._all_slaves_by_url = {}
self._all_builds_by_id = OrderedDict()
self._build_request_handler = BuildRequestHandler()
self._build_request_handler.start()
self._slave_allocator = SlaveAllocator(self._build_request_handler)
self._slave_allocator.start()
# Asynchronously delete (but immediately rename) all old builds when master starts.
# Remove this if/when build numbers are unique across master starts/stops
if os.path.exists(self._master_results_path):
fs.async_delete(self._master_results_path)
fs.create_dir(self._master_results_path)
