def get_check_itasks_by_non_empty_iset_number_from_loaded_isc_slices(self):
        ne_iset_number = self.working_ne_iset_numbers
        task_slices = self.incremental_task_slices[ne_iset_number]
        unknown_iset_number = len(self.meta_data.se_iset_ids)
        se_iset_ids = self.meta_data.se_iset_ids

        filtered_task_slices = list()
        check_icondition_number = 0

        for ts in task_slices:
            task_counter = CombinaryCounter(ne_iset_number,
                                            unknown_iset_number)
            task_counter.reset_current_indicator(ts[0])
            task_begin = copy.deepcopy(ts[0])
            task_length = 0

            for i in range(ts[1]):
                task_idx = task_counter.get_current_indicator()
                if task_length == 0:
                    task_begin = copy.deepcopy(task_idx)

                iset_ids = set()
                for t in task_idx:
                    iset_ids.add(se_iset_ids[t])

                if not self.is_contain_non_se_condition(iset_ids):
                    task_length += 1
                else:
                    if task_length > 0:
                        filtered_task_slices.append((task_begin, task_length))
                        check_icondition_number += task_length
                        task_length = 0
                    # task_idx = task_counter.get_current_indicator()
                    # task_begin = copy.deepcopy(task_idx)
            if task_length > 0:
                filtered_task_slices.append((task_begin, task_length))
                check_icondition_number += task_length

            nse_icondition_number = self.incremental_task_number[
                ne_iset_number] - check_icondition_number
            self.incremental_task_check_number[
                ne_iset_number] = check_icondition_number
            self.incremental_task_complete_number[
                ne_iset_number] = nse_icondition_number
            self.incremental_nse_condition_number[
                ne_iset_number] = nse_icondition_number
        msg_text = "task queue put %d itasks" % len(filtered_task_slices)
        return filtered_task_slices, msg_text
def kmn_isc_task_worker(isc_config_file="isets-tasks.json", worker_name="", lp_type="lpmln", is_check_valid_rules=True, is_use_extended_rules=True):

    ISCFileTaskTerminationWorkerQueueManager.register("get_task_queue")
    ISCFileTaskTerminationWorkerQueueManager.register("get_result_queue")
    manager = ISCFileTaskTerminationWorkerQueueManager(address=(config.task_host, config.task_host_port),
                                                       authkey=bytes(config.task_host_key, encoding="utf-8"))
    manager.connect()
    task_queue = manager.get_task_queue()
    result_queue = manager.get_result_queue()

    time_fmt = "%Y-%m-%d %H:%M:%S.%f"
    worker_host_name = config.worker_host_name
    msg_text = "task worker %s start!" % (worker_name)
    logging.info(msg_text)
    isc_tasks = ISCTaskConfig(isc_config_file, is_use_extended_rules)
    isc_tasks = isc_tasks.isc_tasks
    processed_task_slices_number = 0

    while True:
        if not pathlib.Path(config.task_host_lock_file).exists():
            break

        if task_queue.empty():
            time.sleep(20)
            continue

        itask = task_queue.get()
        if itask[0] == kill_signal:
            msg_text = "%s:%s isc task worker terminate ..." % (worker_host_name, worker_name)
            logging.info(msg_text)
            break

        start_time = datetime.now()
        start_time_str = start_time.strftime(time_fmt)[:-3]

        isc_task_id = itask[0]
        it = isc_tasks[isc_task_id]
        k_size = it.k_m_n[0]
        m_size = it.k_m_n[1]
        n_size = it.k_m_n[2]

        se_iset_ids = it.meta_data.se_iset_ids
        unknown_iset_number = len(se_iset_ids)
        task_details = itask[1]
        isc_begin = copy.deepcopy(task_details[0])

        task_start = task_details[0]
        task_start = [str(s) for s in task_start]
        task_start = ",".join(task_start)

        task_number = task_details[1]
        task_name = worker_name + ("-task-%d" % processed_task_slices_number)
        ne_number = len(isc_begin)

        msg_text = "%s: %d-%d-%d isc task: from %s length %d, nonempty iset number %d" % (
            task_name, k_size, m_size, n_size, task_start, task_number, ne_number)
        logging.info(msg_text)

        task_counter = CombinaryCounter(ne_number, unknown_iset_number)
        task_counter.reset_current_indicator(isc_begin)
        se_cdt_cnt = 0
        nse_cdt_cnt = 0

        se_conditions_cache = list()
        nse_conditions_cache = list()
        validator = ISetConditionValidator(lp_type=lp_type, is_use_extended_rules=is_use_extended_rules)

        for i in range(task_number):
            task_idx = task_counter.get_current_indicator()
            non_ne_ids = set()
            for t in task_idx:
                non_ne_ids.add(se_iset_ids[t])

            is_contain_valid_rule, is_strongly_equivalent, condition = \
                validator.validate_kmn_extended_iset_condition_from_non_emtpy_iset_ids_return_icondition_obj(
                    non_ne_ids, k_size, m_size, n_size, is_check_valid_rule=is_check_valid_rules)

            if not is_contain_valid_rule:
                if is_strongly_equivalent:
                    se_conditions_cache.append(condition)
                    se_cdt_cnt += 1
                else:
                    nse_conditions_cache.append(condition)
                    nse_cdt_cnt += 1

        # for sec in se_conditions_cache:
        result_queue.put((nse_condition_signal, isc_task_id, nse_conditions_cache))
        result_queue.put((se_condition_signal, isc_task_id, se_conditions_cache))

        end_time = datetime.now()
        end_time_str = end_time.strftime(time_fmt)[:-3]
        msg_text = "%s, end %d-%d-%d isc tasks from %s length %d, start time %s, end time %s, find %d se conditions (no semi-valid rules), find %d non-se conditions" % (
            task_name, k_size, m_size, n_size, task_start, task_number, start_time_str, end_time_str, se_cdt_cnt, nse_cdt_cnt)
        logging.info(msg_text)
        result_queue.put((stat_signal, isc_task_id, ne_number, task_number, (start_time, end_time)))
        processed_task_slices_number += 1

    logging.info("%s processes %d isc task slices" % (worker_name, processed_task_slices_number))