def get_check_itasks_by_non_empty_iset_number_from_loaded_isc_slices(self): ne_iset_number = self.working_ne_iset_numbers task_slices = self.incremental_task_slices[ne_iset_number] unknown_iset_number = len(self.meta_data.se_iset_ids) se_iset_ids = self.meta_data.se_iset_ids filtered_task_slices = list() check_icondition_number = 0 for ts in task_slices: task_counter = CombinaryCounter(ne_iset_number, unknown_iset_number) task_counter.reset_current_indicator(ts[0]) task_begin = copy.deepcopy(ts[0]) task_length = 0 for i in range(ts[1]): task_idx = task_counter.get_current_indicator() if task_length == 0: task_begin = copy.deepcopy(task_idx) iset_ids = set() for t in task_idx: iset_ids.add(se_iset_ids[t]) if not self.is_contain_non_se_condition(iset_ids): task_length += 1 else: if task_length > 0: filtered_task_slices.append((task_begin, task_length)) check_icondition_number += task_length task_length = 0 # task_idx = task_counter.get_current_indicator() # task_begin = copy.deepcopy(task_idx) if task_length > 0: filtered_task_slices.append((task_begin, task_length)) check_icondition_number += task_length nse_icondition_number = self.incremental_task_number[ ne_iset_number] - check_icondition_number self.incremental_task_check_number[ ne_iset_number] = check_icondition_number self.incremental_task_complete_number[ ne_iset_number] = nse_icondition_number self.incremental_nse_condition_number[ ne_iset_number] = nse_icondition_number msg_text = "task queue put %d itasks" % len(filtered_task_slices) return filtered_task_slices, msg_text
def kmn_isc_task_worker(isc_config_file="isets-tasks.json", worker_name="", lp_type="lpmln", is_check_valid_rules=True, is_use_extended_rules=True): ISCFileTaskTerminationWorkerQueueManager.register("get_task_queue") ISCFileTaskTerminationWorkerQueueManager.register("get_result_queue") manager = ISCFileTaskTerminationWorkerQueueManager(address=(config.task_host, config.task_host_port), authkey=bytes(config.task_host_key, encoding="utf-8")) manager.connect() task_queue = manager.get_task_queue() result_queue = manager.get_result_queue() time_fmt = "%Y-%m-%d %H:%M:%S.%f" worker_host_name = config.worker_host_name msg_text = "task worker %s start!" % (worker_name) logging.info(msg_text) isc_tasks = ISCTaskConfig(isc_config_file, is_use_extended_rules) isc_tasks = isc_tasks.isc_tasks processed_task_slices_number = 0 while True: if not pathlib.Path(config.task_host_lock_file).exists(): break if task_queue.empty(): time.sleep(20) continue itask = task_queue.get() if itask[0] == kill_signal: msg_text = "%s:%s isc task worker terminate ..." % (worker_host_name, worker_name) logging.info(msg_text) break start_time = datetime.now() start_time_str = start_time.strftime(time_fmt)[:-3] isc_task_id = itask[0] it = isc_tasks[isc_task_id] k_size = it.k_m_n[0] m_size = it.k_m_n[1] n_size = it.k_m_n[2] se_iset_ids = it.meta_data.se_iset_ids unknown_iset_number = len(se_iset_ids) task_details = itask[1] isc_begin = copy.deepcopy(task_details[0]) task_start = task_details[0] task_start = [str(s) for s in task_start] task_start = ",".join(task_start) task_number = task_details[1] task_name = worker_name + ("-task-%d" % processed_task_slices_number) ne_number = len(isc_begin) msg_text = "%s: %d-%d-%d isc task: from %s length %d, nonempty iset number %d" % ( task_name, k_size, m_size, n_size, task_start, task_number, ne_number) logging.info(msg_text) task_counter = CombinaryCounter(ne_number, unknown_iset_number) task_counter.reset_current_indicator(isc_begin) se_cdt_cnt = 0 nse_cdt_cnt = 0 se_conditions_cache = list() nse_conditions_cache = list() validator = ISetConditionValidator(lp_type=lp_type, is_use_extended_rules=is_use_extended_rules) for i in range(task_number): task_idx = task_counter.get_current_indicator() non_ne_ids = set() for t in task_idx: non_ne_ids.add(se_iset_ids[t]) is_contain_valid_rule, is_strongly_equivalent, condition = \ validator.validate_kmn_extended_iset_condition_from_non_emtpy_iset_ids_return_icondition_obj( non_ne_ids, k_size, m_size, n_size, is_check_valid_rule=is_check_valid_rules) if not is_contain_valid_rule: if is_strongly_equivalent: se_conditions_cache.append(condition) se_cdt_cnt += 1 else: nse_conditions_cache.append(condition) nse_cdt_cnt += 1 # for sec in se_conditions_cache: result_queue.put((nse_condition_signal, isc_task_id, nse_conditions_cache)) result_queue.put((se_condition_signal, isc_task_id, se_conditions_cache)) end_time = datetime.now() end_time_str = end_time.strftime(time_fmt)[:-3] msg_text = "%s, end %d-%d-%d isc tasks from %s length %d, start time %s, end time %s, find %d se conditions (no semi-valid rules), find %d non-se conditions" % ( task_name, k_size, m_size, n_size, task_start, task_number, start_time_str, end_time_str, se_cdt_cnt, nse_cdt_cnt) logging.info(msg_text) result_queue.put((stat_signal, isc_task_id, ne_number, task_number, (start_time, end_time))) processed_task_slices_number += 1 logging.info("%s processes %d isc task slices" % (worker_name, processed_task_slices_number))