def tune_parallel_jobs_num(self): """Tune the jobs num. """ user_jobs_num = self.__options.jobs jobs_num = 0 cpu_core_num = cpu_count() distcc_enabled = configparse.blade_config.get_config( 'distcc_config')['enabled'] if distcc_enabled and self.build_environment.distcc_env_prepared: jobs_num = int( 1.5 * len(self.build_environment.get_distcc_hosts_list())) + 1 if jobs_num > 20: jobs_num = 20 if jobs_num and self.__options.jobs != jobs_num: self.__options.jobs = jobs_num elif self.__options.jobs < 1: if cpu_core_num <= 4: self.__options.jobs = 2 * cpu_core_num else: self.__options.jobs = cpu_core_num if self.__options.jobs > 8: self.__options.jobs = 8 if self.__options.jobs != user_jobs_num: console.info('tunes the parallel jobs number(-j N) to be %d' % (self.__options.jobs)) return self.__options.jobs
def parallel_jobs_num(self): """Tune the jobs num. """ # User has the highest priority user_jobs_num = self.__options.jobs if user_jobs_num > 0: return user_jobs_num # Calculate job numbers smartly jobs_num = 0 distcc_enabled = configparse.blade_config.get_config('distcc_config')['enabled'] if distcc_enabled and self.build_environment.distcc_env_prepared: # Distcc cost doesn;t much local cpu, jobs can be quite large. distcc_num = len(self.build_environment.get_distcc_hosts_list()) jobs_num = min(max(int(1.5 * distcc_num), 1), 20) else: cpu_core_num = cpu_count() # machines with cpu_core_num > 4 is usually shared by multiple users, # set an upper bound to avoid interfering other users jobs_num = min(2 * cpu_core_num, 8) if jobs_num != user_jobs_num: console.info('tunes the parallel jobs number(-j N) to be %d' % ( jobs_num)) return jobs_num
def _get_workers_num(self): """get the number of thread workers. """ cpu_count = blade_util.cpu_count() max_workers = max(cpu_count, _MAX_WORKER_THREADS) if self.num_jobs <= 1: return 1 elif self.num_jobs > max_workers: self.num_jobs = max_workers return min(len(self.tests_list), self.num_jobs)
def __init__(self, tests_list, jobs, tests_run_map): """init method. """ self.tests_list = tests_list self.jobs = jobs self.tests_run_map = tests_run_map self.tests_run_map_lock = threading.Lock() self.cpu_core_num = blade_util.cpu_count() self.num_of_tests = len(self.tests_list) self.max_worker_threads = 16 self.failed_targets = [] self.failed_targets_lock = threading.Lock() self.num_of_run_tests = 0 self.num_of_run_tests_lock = threading.Lock() self.job_queue = Queue.Queue(0) self.exclusive_job_queue = Queue.Queue(0)
def tune_parallel_jobs_num(self): """Tune the jobs num. """ user_jobs_num = self.options.jobs jobs_num = 0 cpu_core_num = cpu_count() if self.distcc_enabled and self.build_environment.distcc_env_prepared: jobs_num = int(1.5 * len(self.build_environment.get_distcc_hosts_list())) + 1 if jobs_num > 20: jobs_num = 20 if jobs_num and self.options.jobs != jobs_num: self.options.jobs = jobs_num elif self.options.jobs < 1: if cpu_core_num <= 4: self.options.jobs = 2 * cpu_core_num else: self.options.jobs = cpu_core_num if self.options.jobs > 8: self.options.jobs = 8 if self.options.jobs != user_jobs_num: console.info("tunes the parallel jobs number(-j N) to be %d" % ( self.options.jobs)) return self.options.jobs