def itask_slices_generator(cls, isc_config_file): max_space_size = 10000000000 msg_text = "%s init task slices generator ..." % str(cls) logging.info(msg_text) msg.send_message(msg_text) manager_tuple = SearchQueueManager.init_task_worker_queue_manager() task_queue = manager_tuple[1] isc_tasks_cfg = ITaskConfig(isc_config_file) isc_tasks = isc_tasks_cfg.isc_tasks for tid in range(len(isc_tasks)): it = isc_tasks[tid] min_ne = it.min_ne max_ne = it.max_ne rule_number = sum(it.k_m_n) isnse.clear_task_space_layer_finish_flag_files( *it.k_m_n, min_ne, max_ne) for ne_iset_number in range(min_ne, max_ne + 1): msg_text = "generating %d-%d-%d %d layer task slices" % ( *it.k_m_n, ne_iset_number) logging.info(msg_text) cls.itask_slice_generator_by_i4_meta(ne_iset_number, tid, it, max_space_size, manager_tuple) # if ne_iset_number <= rule_number: # cls.itask_slice_generator_by_i4_meta(ne_iset_number, tid, it, max_space_size, manager_tuple) # else: # if not cls.check_itask_terminate_status(it): # flag_file = isnse.get_task_space_layer_finish_flag_file(*it.k_m_n, ne_iset_number - 2) # while not pathlib.Path(flag_file).exists(): # if cls.check_itask_terminate_status(it): # break # time.sleep(1) # # cls.itask_slice_generator_by_i4_meta(ne_iset_number, tid, it, max_space_size, manager_tuple) working_hosts_number = 5 for i in range(working_hosts_number * 200): task_queue.put((ITaskSignal.kill_signal, -1)) logging.info("all itasks has been dispatched")
def itask_slices_generator(cls, isc_config_file): max_space_size = 100000000000 msg_text = "%s init task slices generator ..." % str(cls) logging.info(msg_text) msg.send_message(msg_text) manager, task_queue, ht_task_queue, result_queue = \ SearchQueueManager.init_task_worker_queue_manager() isc_tasks_cfg = ITaskConfig(isc_config_file) isc_tasks = isc_tasks_cfg.isc_tasks for tid in range(len(isc_tasks)): it = isc_tasks[tid] min_ne = it.min_ne max_ne = it.max_ne isnse.clear_task_space_layer_finish_flag_files(*it.k_m_n, min_ne, max_ne) left_zone_length = len(it.meta_data.search_i4_composed_iset_ids) search_isets = copy.deepcopy(it.meta_data.search_space_iset_ids) search_isets_length = len(search_isets) max_left_zone_length = 12 if left_zone_length > max_left_zone_length: left_zone_length = 12 rule_number = sum(it.k_m_n) left_zone_iset_ids = search_isets[0:left_zone_length] right_zone_iset_ids = search_isets[left_zone_length:] for ne_iset_number in range(min_ne, max_ne + 1): msg_text = "generating %d-%d-%d %d layer task slices" % (*it.k_m_n, ne_iset_number) logging.info(msg_text) if ne_iset_number <= rule_number: left_split = True task_slices = CombinationSearchingSpaceSplitter.vandermonde_generator( left_zone_iset_ids, right_zone_iset_ids, ne_iset_number) for ts in task_slices: new_ts = (left_split, set(ts[0]), left_zone_length, ts[2]) task_queue.put((tid, new_ts)) else: if not cls.check_itask_terminate_status(it): flag_file = isnse.get_task_space_layer_finish_flag_file(*it.k_m_n, ne_iset_number - 2) while not pathlib.Path(flag_file).exists(): if cls.check_itask_terminate_status(it): break time.sleep(1) task_slices = CombinationSearchingSpaceSplitter.merge_small_near_uniform_vandermonde_generator( left_zone_iset_ids, right_zone_iset_ids, ne_iset_number, max_space_size=max_space_size) ts_cnt = 0 for ts in task_slices: task_queue.put((tid, ts)) ts_cnt += 1 if ts_cnt % 10000 == 0 and cls.check_itask_terminate_status(it): break working_hosts_number = 5 for i in range(working_hosts_number * 200): task_queue.put((ITaskSignal.kill_signal, -1)) logging.info("all itasks has been dispatched")