def process_result_queue(result_queue, isc_tasks):
    working_hosts_number_diff = (0, 0)
    result = result_queue.get()
    result_state = result[0]
    isc_task_id = result[1]

    if result_state == kill_signal:
        host_ip = result[2]
        working_hosts_number_diff = (-1, host_ip)
        msg_text = "task host %s:%s exit" % (isc_task_id, host_ip)
        logging.info(msg_text)
        msg.send_message(msg_text)
    elif result_state == add_worker_signal:
        host_ip = result[2]
        msg_text = "task host %s:%s is online" % (isc_task_id, host_ip)
        working_hosts_number_diff = (1, host_ip)
        logging.info(msg_text)
        msg.send_message(msg_text)
    elif result_state == stat_signal:
        ne_iset_number = result[2]
        task_complete_number = result[3]
        task_running_time = result[4]
        isc_tasks[isc_task_id].set_task_complete_number(task_complete_number, ne_iset_number)
        isc_tasks[isc_task_id].set_task_running_time(task_running_time)
    elif result_state == se_condition_signal:
        iconditions = result[2]
        for ic in iconditions:
            isc_tasks[isc_task_id].insert_se_condition(ic)
            # put_isc_task_items(isc_task_id, itask_items, msg_text, task_queue)
    elif result_state == nse_condition_signal:
        iconditions = result[2]
        for ic in iconditions:
            isc_tasks[isc_task_id].insert_nse_condition(ic)

    return working_hosts_number_diff
Esempio n. 2
0
def process_result_queue(task_queue, result_queue, isc_tasks):
    working_hosts_number_diff = 0
    result = result_queue.get()
    result_state = result[0]
    isc_task_id = result[1]

    if result_state == kill_signal:
        working_hosts_number_diff = -1
        msg_text = "task host %s exit" % isc_task_id
        logging.info(msg_text)
        msg.send_message(msg_text)
    elif result_state == add_worker_signal:
        msg_text = "task host %s is online" % isc_task_id
        working_hosts_number_diff = 1
        logging.info(msg_text)
        msg.send_message(msg_text)
    elif result_state == stat_signal:
        ne_iset_number = result[2]
        task_complete_number = result[3]
        task_running_time = result[4]
        isc_tasks[isc_task_id].set_task_complete_number(
            task_complete_number, ne_iset_number)
        isc_tasks[isc_task_id].set_task_running_time(task_running_time)
    elif result_state == condition_signal:
        iconditions = result[2]
        for ic in iconditions:
            if ic.contain_se_valid_rules:
                itask_items, msg_text = isc_tasks[
                    isc_task_id].generate_isc_task_items_by_base_icondition(ic)
            else:
                itask_items, msg_text = isc_tasks[
                    isc_task_id].insert_se_condition(ic)
            put_isc_task_items(isc_task_id, itask_items, msg_text, task_queue)

    return working_hosts_number_diff
def check_itasks_status(itasks, task_queue, working_hosts_number, is_frequent_log):
    is_finish = True
    is_task_queue_change = False
    for tid in range(len(itasks)):
        it = itasks[tid]
        if not it.is_task_finish:
            current_ne_number = it.working_ne_iset_numbers
            task_complete = it.incremental_task_complete_number[current_ne_number]
            task_total = it.incremental_task_number[current_ne_number]
            if task_complete == task_total:
                if it.is_early_terminate():
                    continue

                if current_ne_number < it.max_ne:
                    is_task_queue_change = True
                    is_finish = False
                    it.flush_non_se_condition()
                    task_slices, msg_text = it.get_check_itasks_by_non_empty_iset_number()

                    for ts in task_slices:
                        task_queue.put((tid, ts))
                    logging.info(msg_text)
                    msg.send_message(msg_text)
                else:
                    it.is_task_finish = True
            else:
                is_finish = False

    if is_task_queue_change and is_frequent_log:
        send_itasks_progress_info(itasks, task_queue, working_hosts_number)

    return is_finish
Esempio n. 4
0
    def itask_slices_generator(cls, isc_config_file):
        msg_text = "%s init task slices generator ..." % str(cls)
        logging.info(msg_text)
        msg.send_message(msg_text)

        manager, task_queue, ht_task_queue, result_queue = \
            SearchQueueManager.init_task_worker_queue_manager()

        isc_tasks_cfg = ITaskConfig(isc_config_file)
        isc_tasks = isc_tasks_cfg.isc_tasks

        for tid in range(len(isc_tasks)):
            it = isc_tasks[tid]
            ht_task_number = it.ht_task_number
            batch_size = 100
            ht_task_slices = [
                i for i in range(0, ht_task_number + 1, batch_size)
            ]
            if ht_task_slices[-1] < ht_task_number:
                ht_task_slices.append(ht_task_number)

            for i in range(1, len(ht_task_slices)):
                ts = (tid, (ht_task_slices[i - 1], ht_task_slices[i]))
                ht_task_queue.put(ts)

        working_hosts_number = 5
        for i in range(working_hosts_number * 200):
            ht_task_queue.put((ITaskSignal.kill_signal, -1))
        logging.info("all itasks has been dispatched")
        msg.send_message("all itasks has been dispatched")
Esempio n. 5
0
 def insert_found_conditions(itask, iconditions, is_se_condition=True):
     if is_se_condition:
         for ic in iconditions:
             itask.insert_se_condition(ic)
     else:
         for ic in iconditions:
             msg_text = "%d-%d-%d nse condition: %s" % (*itask.k_m_n,
                                                        str(ic))
             logging.error(msg_text)
             msg.send_message(msg_text)
Esempio n. 6
0
 def process_working_host_change(info, is_add=True):
     host_name = info[1]
     host_ip = info[2]
     if is_add:
         diff = 1
         msg_text = "task host %s:%s is online" % (host_name, host_ip)
     else:
         diff = -1
         msg_text = "task host %s:%s exit" % (host_name, host_ip)
     working_hosts_number_diff = (diff, host_ip)
     logging.info(msg_text)
     msg.send_message(msg_text)
     return working_hosts_number_diff
Esempio n. 7
0
    def itask_slices_generator(cls, isc_config_file="isets-tasks.json"):
        msg_text = "%s init task slices generator ..." % str(cls)
        logging.info(msg_text)
        msg.send_message(msg_text)

        SearchWorkerQueueManger.register("get_task_queue")
        SearchWorkerQueueManger.register("get_result_queue")
        manager = SearchWorkerQueueManger(address=(config.task_host,
                                                   config.task_host_port),
                                          authkey=bytes(config.task_host_key,
                                                        encoding="utf-8"))
        manager.connect()
        task_queue = manager.get_task_queue()

        isc_tasks_cfg = ITaskConfig(isc_config_file)
        isc_tasks = isc_tasks_cfg.isc_tasks

        for tid in range(len(isc_tasks)):
            it = isc_tasks[tid]
            min_ne = it.min_ne
            max_ne = it.max_ne
            search_iset_ids = it.meta_data.search_space_iset_ids
            unknown_iset_number = len(search_iset_ids)
            left_length = int(unknown_iset_number / 2)
            if left_length > 12:
                left_length = 12

            right_length = unknown_iset_number - left_length
            left_zone_isets = search_iset_ids[0:left_length]

            for i in range(min_ne, max_ne + 1):
                ne_iset_number = i
                for left_iset_number in range(ne_iset_number + 1):
                    right_iset_number = ne_iset_number - left_iset_number
                    if left_iset_number > left_length or right_iset_number > right_length:
                        continue

                    task_iter = itertools.combinations(left_zone_isets,
                                                       left_iset_number)
                    for left_ti in task_iter:
                        task_item = (tid,
                                     (ne_iset_number, set(left_zone_isets),
                                      list(left_ti)))
                        # print(task_item)
                        task_queue.put(task_item)

        working_hosts_number = 5
        for i in range(working_hosts_number * 200):
            task_queue.put((ITaskSignal.kill_signal, -1))
        logging.info("all itasks has been dispatched")
Esempio n. 8
0
 def send_itasks_progress_info(cls,
                               isc_tasks,
                               task_queue,
                               working_hosts_number,
                               is_all_task_dispatched=False):
     msg_texts = cls.dump_isc_task_results(isc_tasks)
     if is_all_task_dispatched:
         msg_text = "all isc tasks are discatched, DO NOT add new worker! isc tasks progress info, remain %d task hosts:  \n\t\t%s" \
                    % (working_hosts_number, "\n\t\t".join(msg_texts))
     else:
         msg_text = "isc tasks progress info, remain %d task hosts, %d task slices:  \n\t\t%s" % (
             working_hosts_number, task_queue.qsize(),
             "\n\t\t".join(msg_texts))
     logging.info(msg_text)
     msg.send_message(msg_text)
Esempio n. 9
0
    def itask_slices_generator(cls, isc_config_file):
        msg_text = "%s init task slices generator ..." % str(cls)
        logging.info(msg_text)
        msg.send_message(msg_text)

        manager, task_queue, ht_task_queue, result_queue = \
            SearchQueueManager.init_task_worker_queue_manager()

        isc_tasks_cfg = ITaskConfig(isc_config_file)
        isc_tasks = isc_tasks_cfg.isc_tasks

        for tid in range(len(isc_tasks)):
            it = isc_tasks[tid]
            min_ne = it.min_ne
            max_ne = it.max_ne

            right_zone_iset_ids = set(
                copy.deepcopy(it.meta_data.search_space_iset_ids))
            left_zone_iset_ids = set(it.meta_data.search_i4_composed_iset_ids)

            max_left_zone_length = 12
            if len(left_zone_iset_ids) > max_left_zone_length:
                left_zone_iset_ids = list(
                    left_zone_iset_ids)[0:max_left_zone_length]
                left_zone_iset_ids = set(left_zone_iset_ids)

            right_zone_iset_ids = right_zone_iset_ids.difference(
                left_zone_iset_ids)

            for ne_iset_number in range(min_ne, max_ne + 1):
                if not cls.check_itask_terminate_status(it):
                    task_slices = CombinationSearchingSpaceSplitter.near_uniform_vandermonde_generator(
                        left_zone_iset_ids, right_zone_iset_ids,
                        ne_iset_number)
                    ts_cnt = 0
                    for ts in task_slices:
                        new_ts = (set(ts[0]), set(ts[1]), ts[2])
                        task_queue.put((tid, new_ts))

                        ts_cnt += 1
                        if ts_cnt % 10000 == 0 and cls.check_itask_terminate_status(
                                it):
                            break

        working_hosts_number = 5
        for i in range(working_hosts_number * 200):
            task_queue.put((ITaskSignal.kill_signal, -1))
        logging.info("all itasks has been dispatched")
Esempio n. 10
0
    def itask_slices_generator(cls, isc_config_file):
        max_space_size = 10000000000
        msg_text = "%s init task slices generator ..." % str(cls)
        logging.info(msg_text)
        msg.send_message(msg_text)

        manager_tuple = SearchQueueManager.init_task_worker_queue_manager()
        task_queue = manager_tuple[1]

        isc_tasks_cfg = ITaskConfig(isc_config_file)
        isc_tasks = isc_tasks_cfg.isc_tasks

        for tid in range(len(isc_tasks)):
            it = isc_tasks[tid]
            min_ne = it.min_ne
            max_ne = it.max_ne
            rule_number = sum(it.k_m_n)
            isnse.clear_task_space_layer_finish_flag_files(
                *it.k_m_n, min_ne, max_ne)

            for ne_iset_number in range(min_ne, max_ne + 1):
                msg_text = "generating %d-%d-%d %d layer task slices" % (
                    *it.k_m_n, ne_iset_number)
                logging.info(msg_text)

                cls.itask_slice_generator_by_i4_meta(ne_iset_number, tid, it,
                                                     max_space_size,
                                                     manager_tuple)

                # if ne_iset_number <= rule_number:
                #     cls.itask_slice_generator_by_i4_meta(ne_iset_number, tid, it, max_space_size, manager_tuple)
                # else:
                #     if not cls.check_itask_terminate_status(it):
                #         flag_file = isnse.get_task_space_layer_finish_flag_file(*it.k_m_n, ne_iset_number - 2)
                #         while not pathlib.Path(flag_file).exists():
                #             if cls.check_itask_terminate_status(it):
                #                 break
                #             time.sleep(1)
                #
                #         cls.itask_slice_generator_by_i4_meta(ne_iset_number, tid, it, max_space_size, manager_tuple)

        working_hosts_number = 5
        for i in range(working_hosts_number * 200):
            task_queue.put((ITaskSignal.kill_signal, -1))
        logging.info("all itasks has been dispatched")
Esempio n. 11
0
    def send_itasks_progress_info(cls,
                                  isc_tasks,
                                  manager_tuple,
                                  working_hosts_number,
                                  is_all_task_dispatched=False):

        task_queue = manager_tuple[1]
        ht_task_queue = manager_tuple[2]
        result_queue = manager_tuple[3]
        msg_texts = cls.dump_isc_task_results(isc_tasks)

        msg_text = "isc tasks progress info, remain %d task hosts, %d task slices, %d ht task slices, %d results items:  \n\t\t%s" % (
            working_hosts_number, task_queue.qsize(), ht_task_queue.qsize(),
            result_queue.qsize(), "\n\t\t".join(msg_texts))

        if is_all_task_dispatched:
            msg_text = "all isc tasks are discatched, DO NOT add new worker! " + msg_text

        logging.info(msg_text)
        msg.send_message(msg_text)
def process_result_queue(result_queue, isc_tasks):
    working_hosts_number_diff = 0
    result = result_queue.get()
    result_state = result[0]
    isc_task_id = result[1]

    if result_state == kill_signal:
        working_hosts_number_diff = -1
        msg_text = "task host %s exit" % isc_task_id
        logging.info(msg_text)
        msg.send_message(msg_text)
    elif result_state == add_worker_signal:
        msg_text = "task host %s is online" % isc_task_id
        working_hosts_number_diff = 1
        logging.info(msg_text)
        msg.send_message(msg_text)
    elif result_state == stat_signal:
        isc_tasks[isc_task_id].task_complete_number += result[2]
        isc_tasks[isc_task_id].set_task_running_time(result[3])
    elif result_state == condition_signal:
        isc_tasks[isc_task_id].insert_se_condition(result[2])

    return working_hosts_number_diff
Esempio n. 13
0
    def itask_slices_generator(cls, isc_config_file):
        msg_text = "%s init task slices generator ..." % str(cls)
        logging.info(msg_text)
        msg.send_message(msg_text)

        manager, task_queue, ht_task_queue, result_queue = \
            SearchQueueManager.init_task_worker_queue_manager()

        isc_tasks_cfg = ITaskConfig(isc_config_file)
        isc_tasks = isc_tasks_cfg.isc_tasks

        for tid in range(len(isc_tasks)):
            it = isc_tasks[tid]
            i4_iset_ids = list(it.meta_data.search_i4_composed_iset_ids)
            i4_iset_size = len(i4_iset_ids)
            min_ne = 1
            max_ne = len(i4_iset_ids)
            max_left_zone_length = 10
            left_zone_size = max_left_zone_length

            if i4_iset_size < left_zone_size:
                left_zone_size = i4_iset_size // 2

            left_zone_iset_ids = i4_iset_ids[0:left_zone_size]
            right_zone_iset_ids = i4_iset_ids[left_zone_size:]

            for ne_iset_number in range(min_ne, max_ne + 1):

                task_slices = CombinationSearchingSpaceSplitter.vandermonde_generator(
                    left_zone_iset_ids, right_zone_iset_ids, ne_iset_number)
                for ts in task_slices:
                    task_queue.put((tid, ts))

        working_hosts_number = 5
        for i in range(working_hosts_number * 200):
            task_queue.put((ITaskSignal.kill_signal, -1))
        logging.info("all itasks has been dispatched")
Esempio n. 14
0
    def send_itasks_progress_info(cls,
                                  result_records,
                                  manager_tuple,
                                  working_hosts_number,
                                  is_all_task_dispatched=False):
        task_queue = manager_tuple[1]
        ht_task_queue = manager_tuple[2]
        result_queue = manager_tuple[3]

        msg_texts = list()
        for i in range(len(result_records)):
            msg_text = "task %d has %d tasks, complete %d tasks" % (
                i, result_records[i][0], result_records[i][1])
            msg_texts.append(msg_text)

        msg_text = "isc tasks progress info, remain %d task hosts, %d task slices, %d ht task slices, %d results items:  \n\t\t%s" % (
            working_hosts_number, task_queue.qsize(), ht_task_queue.qsize(),
            result_queue.qsize(), "\n\t\t".join(msg_texts))

        if is_all_task_dispatched:
            msg_text = "all isc tasks are discatched, DO NOT add new worker! " + msg_text

        logging.info(msg_text)
        msg.send_message(msg_text)
def send_itasks_progress_info(isc_tasks, task_queue, working_hosts_number):
    msg_texts = dump_isc_task_results(isc_tasks)
    msg_text = "isc tasks progress info, remain %d task hosts, %d task slices:  \n\t\t%s" % (
        working_hosts_number, task_queue.qsize(), "\n\t\t".join(msg_texts))
    logging.info(msg_text)
    msg.send_message(msg_text)
Esempio n. 16
0
    def init_kmn_isc_task_master_from_config(
            cls, isc_config_file="isets-tasks.json", sleep_time=30):
        manager, task_queue, ht_task_queue, result_queue = \
            SearchQueueManager.init_task_master_queue_manager()
        manager_tuple = (manager, task_queue, ht_task_queue, result_queue)
        localhost_ip = ssh.get_host_ip()

        isc_tasks_cfg = ITaskConfig(isc_config_file)
        isc_tasks = isc_tasks_cfg.isc_tasks
        result_record = list()
        for itask in isc_tasks:
            isnse.clear_task_terminate_flag_files(*itask.k_m_n)
            i4_iset_size = len(itask.meta_data.search_i4_composed_iset_ids)
            file = i4u.get_kmn_i4_all_result_file(*itask.k_m_n)
            if os.path.exists(file):
                os.remove(file)
            record = [2**i4_iset_size - 1, 0, list(), file]
            result_record.append(record)

        ts_generator_pool = cls.init_task_slices_generator_pool(
            cls, isc_config_file)

        pre_task_pool = cls.init_pre_task_worker_pool(cls, isc_config_file,
                                                      result_queue)

        working_hosts_number = 0

        msg_text = "isc task master start, load %d isc tasks from %s" % (
            len(isc_tasks), isc_config_file)
        logging.info(msg_text)
        msg.send_message(msg_text)

        sleep_cnt = 0
        online_hosts = set()

        progress_msg_cnt = 10
        task_finish = False
        print_loop = 10
        print_cnt = 0
        while not task_finish:
            print_cnt += 1

            if print_cnt == print_loop:
                cls.send_itasks_progress_info(cls, result_record,
                                              manager_tuple,
                                              working_hosts_number, False)
                sleep_cnt = 0
                print_cnt = 0

            if sleep_cnt == progress_msg_cnt:
                cls.send_itasks_progress_info(cls, result_record,
                                              manager_tuple,
                                              working_hosts_number, False)
                sleep_cnt = 0
                print_cnt = 0

            task_finish = cls.check_i4_tasks_status(cls, result_record)
            if result_queue.empty():
                time.sleep(sleep_time)
                sleep_cnt += 1
                continue

            whn_diff = cls.process_i4_result_queue(cls, result_queue,
                                                   result_record)

            whn_number = whn_diff[0]
            host_ip = whn_diff[1]
            working_hosts_number += whn_number

            if whn_number == 1:
                if host_ip != localhost_ip:
                    online_hosts.add(host_ip)
            elif whn_number == -1:
                if host_ip != localhost_ip:
                    online_hosts.remove(host_ip)

        ts_generator_pool.join()
        pre_task_pool.join()

        I4SearchWorker.send_worker_terminate_info(I4SearchWorker, localhost_ip,
                                                  result_queue)

        while working_hosts_number > 0:
            if sleep_cnt == 10:
                cls.send_itasks_progress_info(cls, result_record,
                                              manager_tuple,
                                              working_hosts_number, True)
                sleep_cnt = 0

            if result_queue.empty():
                time.sleep(sleep_time)
                sleep_cnt += 1
                continue

            whn_diff = cls.process_i4_result_queue(cls, result_queue,
                                                   result_record)
            working_hosts_number += whn_diff[0]

        msg_text = "isc tasks finish!"
        logging.info(msg_text)
        msg.send_message(msg=msg_text)
        cls.send_itasks_progress_info(cls, result_record, manager_tuple,
                                      working_hosts_number, True)

        return isc_tasks
def init_kmn_isc_task_master_from_config(isc_config_file="isets-tasks.json", sleep_time=60, is_use_extended_rules=True):
    start_time = datetime.now()
    ISCFileTaskMasterQueueManager.register("get_task_queue", callable=get_task_queue)
    ISCFileTaskMasterQueueManager.register("get_result_queue", callable=get_result_queue)
    manager = ISCFileTaskMasterQueueManager(address=(config.task_host, config.task_host_port),
                                            authkey=bytes(config.task_host_key, encoding="utf-8"))
    manager.start()
    task_queue = manager.get_task_queue()
    result_queue = manager.get_result_queue()

    working_hosts_number = 0
    msg_text = "isc task master start, load isc tasks from %s" % (isc_config_file)
    logging.info(msg_text)
    msg.send_message(msg_text)

    isc_tasks_cfg = isc_cfg.ISCTaskConfig(isc_config_file, is_use_extended_rules)
    isc_tasks = isc_tasks_cfg.isc_tasks

    for isc_id in range(len(isc_tasks)):
        isc_task_slices = isc_tasks[isc_id]
        isc_task_items = isc_task_slices.load_isc_task_items()
        for it in isc_task_items:
            task_queue.put((isc_id, it))
        msg_text = isc_task_slices.get_isc_task_load_message()

        logging.info(msg_text)
        msg.send_message(msg_text)

    sleep_cnt = 0
    while not task_queue.empty():
        if sleep_cnt == 10:
            msg_texts = dump_isc_task_results(isc_tasks)
            msg_text = "isc tasks progress info, remain %d task hosts, %d task slices:  \n\t\t%s" % (
                working_hosts_number, task_queue.qsize(), "\n\t\t".join(msg_texts))
            logging.info(msg_text)
            msg.send_message(msg_text)
            sleep_cnt = 0

        if result_queue.empty():
            time.sleep(sleep_time)
            sleep_cnt += 1
            continue

        whn_diff = process_result_queue(result_queue, isc_tasks)
        working_hosts_number += whn_diff

    msg_text = "all isc task slices are discatched!"
    logging.info(msg_text)
    msg.send_message(msg_text)

    for i in range(working_hosts_number * 200):
        task_queue.put((kill_signal, -1))

    while working_hosts_number > 0:
        if sleep_cnt == 10:
            msg_texts = dump_isc_task_results(isc_tasks)

            msg_text = "all isc tasks are discatched, DO NOT add new worker! isc tasks progress info, remain %d task hosts:  \n\t\t%s" \
                       % (working_hosts_number, "\n\t\t".join(msg_texts))
            logging.info(msg_text)
            msg.send_message(msg_text)
            sleep_cnt = 0

        if result_queue.empty():
            time.sleep(sleep_time)
            sleep_cnt += 1
            continue

        whn_diff = process_result_queue(result_queue, isc_tasks)
        working_hosts_number += whn_diff

    msg_texts = []
    attached_files = []
    for it in isc_tasks:
        msg_texts.append(it.task_finish())
        attached_files.append(it.result_file)

    msg_text = "isc tasks finish! \n\t\t%s" % "\n\t\t".join(msg_texts)
    logging.info(msg_text)
    msg.send_message(msg=msg_text, attached_files=attached_files)
    def itask_slices_generator(cls, isc_config_file):
        max_space_size = 100000000000
        msg_text = "%s init task slices generator ..." % str(cls)
        logging.info(msg_text)
        msg.send_message(msg_text)

        manager, task_queue, ht_task_queue, result_queue = \
            SearchQueueManager.init_task_worker_queue_manager()

        isc_tasks_cfg = ITaskConfig(isc_config_file)
        isc_tasks = isc_tasks_cfg.isc_tasks

        for tid in range(len(isc_tasks)):
            it = isc_tasks[tid]
            min_ne = it.min_ne
            max_ne = it.max_ne
            isnse.clear_task_space_layer_finish_flag_files(*it.k_m_n, min_ne, max_ne)

            left_zone_length = len(it.meta_data.search_i4_composed_iset_ids)
            search_isets = copy.deepcopy(it.meta_data.search_space_iset_ids)
            search_isets_length = len(search_isets)

            max_left_zone_length = 12
            if left_zone_length > max_left_zone_length:
                left_zone_length = 12

            rule_number = sum(it.k_m_n)
            left_zone_iset_ids = search_isets[0:left_zone_length]
            right_zone_iset_ids = search_isets[left_zone_length:]

            for ne_iset_number in range(min_ne, max_ne + 1):
                msg_text = "generating %d-%d-%d %d layer task slices" % (*it.k_m_n, ne_iset_number)
                logging.info(msg_text)
                if ne_iset_number <= rule_number:
                    left_split = True
                    task_slices = CombinationSearchingSpaceSplitter.vandermonde_generator(
                        left_zone_iset_ids, right_zone_iset_ids, ne_iset_number)
                    for ts in task_slices:
                        new_ts = (left_split, set(ts[0]), left_zone_length, ts[2])
                        task_queue.put((tid, new_ts))
                else:
                    if not cls.check_itask_terminate_status(it):
                        flag_file = isnse.get_task_space_layer_finish_flag_file(*it.k_m_n, ne_iset_number - 2)
                        while not pathlib.Path(flag_file).exists():
                            if cls.check_itask_terminate_status(it):
                                break
                            time.sleep(1)

                        task_slices = CombinationSearchingSpaceSplitter.merge_small_near_uniform_vandermonde_generator(
                            left_zone_iset_ids, right_zone_iset_ids, ne_iset_number, max_space_size=max_space_size)
                        ts_cnt = 0
                        for ts in task_slices:
                            task_queue.put((tid, ts))

                            ts_cnt += 1
                            if ts_cnt % 10000 == 0 and cls.check_itask_terminate_status(it):
                                break

        working_hosts_number = 5
        for i in range(working_hosts_number * 200):
            task_queue.put((ITaskSignal.kill_signal, -1))
        logging.info("all itasks has been dispatched")
    def init_kmn_isc_task_master_from_config(cls, isc_config_file="isets-tasks.json", sleep_time=30):
        start_time = datetime.now()
        manager, task_queue, ht_task_queue, result_queue = \
            SearchQueueManager.init_task_master_queue_manager()
        manager_tuple = (manager, task_queue, ht_task_queue, result_queue)
        localhost_ip = ssh.get_host_ip()

        isc_tasks_cfg = ITaskConfig(isc_config_file)
        isc_tasks = isc_tasks_cfg.isc_tasks
        for itask in isc_tasks:
            itask.init_task_numbers()
            isnse.clear_task_terminate_flag_files(*itask.k_m_n)

        ts_generator_pool = cls.init_task_slices_generator_pool(cls, isc_config_file)
        pre_pool = cls.init_pre_task_worker_pool(cls, isc_config_file, result_queue)
        working_hosts_number = 0

        msg_text = "isc task master start, load %d isc tasks from %s" % (len(isc_tasks), isc_config_file)
        logging.info(msg_text)
        msg.send_message(msg_text)

        sleep_cnt = 0
        online_hosts = set()

        progress_msg_cnt = 10
        task_finish = False
        print_loop = 100
        print_cnt = 0
        while not task_finish:
            print_cnt += 1

            if print_cnt == print_loop:
                cls.send_itasks_progress_info(cls, isc_tasks, manager_tuple, working_hosts_number, False)
                for it in isc_tasks:
                    it.save_progress_info()
                sleep_cnt = 0
                print_cnt = 0

            if sleep_cnt == progress_msg_cnt:
                cls.send_itasks_progress_info(cls, isc_tasks, manager_tuple, working_hosts_number, False)
                for it in isc_tasks:
                    it.save_progress_info()
                sleep_cnt = 0
                print_cnt = 0

            task_finish = cls.check_itasks_status(cls, isc_tasks, online_hosts, manager_tuple, working_hosts_number)
            if result_queue.empty():
                time.sleep(sleep_time)
                sleep_cnt += 1
                continue

            whn_diff = cls.process_result_queue(cls, result_queue, isc_tasks)

            whn_number = whn_diff[0]
            host_ip = whn_diff[1]
            working_hosts_number += whn_number

            if whn_number == 1:
                if host_ip != localhost_ip:
                    online_hosts.add(host_ip)
                    cls.update_nse_files_to_new_host(host_ip, isc_tasks)
            elif whn_number == -1:
                if host_ip != localhost_ip:
                    online_hosts.remove(host_ip)

        ts_generator_pool.join()
        pre_pool.join()
        RawIConditionSearchWorker.send_worker_terminate_info(RawIConditionSearchWorker, localhost_ip, result_queue)

        while working_hosts_number > 0:
            if sleep_cnt == 10:
                cls.send_itasks_progress_info(cls, isc_tasks, manager_tuple, working_hosts_number, True)
                sleep_cnt = 0

            if result_queue.empty():
                time.sleep(sleep_time)
                sleep_cnt += 1
                continue

            whn_diff = cls.process_result_queue(cls, result_queue, isc_tasks)
            working_hosts_number += whn_diff[0]

        end_time = datetime.now()
        msg_text = "isc tasks finish, running time: %s" % str(end_time - start_time)
        logging.info(msg_text)
        msg.send_message(msg=msg_text)

        for it in isc_tasks:
            it.task_finish()
            msg_text = it.get_final_detail_progress_info()
            logging.info(msg_text)
            msg.send_message(msg=msg_text)
            # attached_files.append(it.result_file)

        return isc_tasks
def init_kmn_isc_task_master_from_config(isc_config_file="isets-tasks.json",
                                         sleep_time=30,
                                         is_use_extended_rules=True,
                                         is_frequent_log=False):
    start_time = datetime.now()
    ISCFileTaskTerminationMasterQueueManager.register("get_task_queue",
                                                      callable=get_task_queue)
    ISCFileTaskTerminationMasterQueueManager.register(
        "get_result_queue", callable=get_result_queue)
    manager = ISCFileTaskTerminationMasterQueueManager(
        address=(config.task_host, config.task_host_port),
        authkey=bytes(config.task_host_key, encoding="utf-8"))
    manager.start()
    task_queue = manager.get_task_queue()
    result_queue = manager.get_result_queue()
    localhost_ip = ssh.get_host_ip()

    task_generator = Pool(2)
    task_generator.apply_async(itask_slices_generator,
                               args=(isc_config_file, is_use_extended_rules))
    task_generator.close()

    working_hosts_number = 0
    msg_text = "isc task master start, load isc tasks from %s" % (
        isc_config_file)
    logging.info(msg_text)
    msg.send_message(msg_text)

    isc_tasks_cfg = ISCTaskConfig(isc_config_file, is_use_extended_rules)
    isc_tasks = isc_tasks_cfg.isc_tasks
    for itask in isc_tasks:
        itask.init_task_numbers()

    sleep_cnt = 0
    online_hosts = set()

    progress_msg_cnt = 10
    task_finish = False
    print_loop = 100000
    print_cnt = 0
    while not task_finish:
        print_cnt += 1

        if print_cnt == print_loop:
            send_itasks_progress_info(isc_tasks, task_queue,
                                      working_hosts_number)
            sleep_cnt = 0
            print_cnt = 0

        if sleep_cnt == progress_msg_cnt:
            send_itasks_progress_info(isc_tasks, task_queue,
                                      working_hosts_number)
            sleep_cnt = 0
            print_cnt = 0

        task_finish = check_itasks_status(isc_tasks, online_hosts, task_queue,
                                          working_hosts_number)
        if result_queue.empty():
            time.sleep(sleep_time)
            sleep_cnt += 1
            continue

        whn_diff = process_result_queue(result_queue, isc_tasks)
        whn_number = whn_diff[0]
        host_ip = whn_diff[1]
        working_hosts_number += whn_number

        if whn_number == 1:
            if host_ip != localhost_ip:
                online_hosts.add(host_ip)
                update_nse_files_to_new_host(host_ip, isc_tasks)
        elif whn_number == -1:
            if host_ip != localhost_ip:
                online_hosts.remove(host_ip)

    msg_text = "all isc task slices are discatched!"
    logging.info(msg_text)
    msg.send_message(msg_text)

    task_generator.join()
    while working_hosts_number > 0:
        if sleep_cnt == 10:
            msg_texts = dump_isc_task_results(isc_tasks)

            msg_text = "all isc tasks are discatched, DO NOT add new worker! isc tasks progress info, remain %d task hosts:  \n\t\t%s" \
                       % (working_hosts_number, "\n\t\t".join(msg_texts))
            logging.info(msg_text)
            msg.send_message(msg_text)
            sleep_cnt = 0

        if result_queue.empty():
            time.sleep(sleep_time)
            sleep_cnt += 1
            continue

        whn_diff = process_result_queue(result_queue, isc_tasks)
        working_hosts_number += whn_diff[0]

    msg_texts = []
    attached_files = []
    for it in isc_tasks:
        it.task_finish()
        msg_texts.append(it.get_final_detail_progress_info())
        attached_files.append(it.result_file)

    msg_text = "isc tasks finish! \n\t\t%s" % "\n\t\t".join(msg_texts)
    logging.info(msg_text)
    msg.send_message(msg=msg_text, attached_files=attached_files)
    return isc_tasks
def init_kmn_isc_task_master_from_config(isc_config_file="isets-tasks.json", sleep_time=30, is_use_extended_rules=True, is_frequent_log=False):
    start_time = datetime.now()
    ISCFileTaskTerminationMasterQueueManager.register("get_task_queue", callable=get_task_queue)
    ISCFileTaskTerminationMasterQueueManager.register("get_result_queue", callable=get_result_queue)
    manager = ISCFileTaskTerminationMasterQueueManager(address=(config.task_host, config.task_host_port),
                                                       authkey=bytes(config.task_host_key, encoding="utf-8"))
    manager.start()
    task_queue = manager.get_task_queue()
    result_queue = manager.get_result_queue()

    working_hosts_number = 0
    msg_text = "isc task master start, load isc tasks from %s" % (isc_config_file)
    logging.info(msg_text)
    msg.send_message(msg_text)

    isc_tasks_cfg = ISCTaskConfig(isc_config_file, is_use_extended_rules)
    isc_tasks = isc_tasks_cfg.isc_tasks

    sleep_cnt = 0
    online_hosts = set()

    for tid in range(len(isc_tasks)):
        itask = isc_tasks[tid]
        itask.load_isc_task_items()
        msg_text = itask.get_isc_task_load_message()
        ne_iset = itask.working_ne_iset_numbers
        slices = itask.incremental_task_slices[ne_iset]
        for s in slices:
            task_queue.put((tid, s))
        logging.info(msg_text)
        msg.send_message(msg_text)


    progress_msg_cnt = 10
    task_finish = False
    while not task_finish:
        if sleep_cnt == progress_msg_cnt:
            send_itasks_progress_info(isc_tasks, task_queue, working_hosts_number)
            sleep_cnt = 0

        task_finish = check_itasks_status(isc_tasks, task_queue, working_hosts_number, is_frequent_log)
        if result_queue.empty():
            time.sleep(sleep_time)
            sleep_cnt += 1
            continue

        whn_diff = process_result_queue(result_queue, isc_tasks)
        whn_number = whn_diff[0]
        host_ip = whn_diff[1]
        working_hosts_number += whn_number

        if whn_number == 1:
            online_hosts.add(host_ip)
            # for itask in isc_tasks:
            #     transport_files(host_ip, itask.non_se_condition_files)
        elif whn_number == -1:
            online_hosts.remove(host_ip)




    msg_text = "all isc task slices are discatched!"
    logging.info(msg_text)
    msg.send_message(msg_text)

    for i in range(working_hosts_number * 200):
        task_queue.put((kill_signal, -1))

    while working_hosts_number > 0:
        if sleep_cnt == 10:
            msg_texts = dump_isc_task_results(isc_tasks)

            msg_text = "all isc tasks are discatched, DO NOT add new worker! isc tasks progress info, remain %d task hosts:  \n\t\t%s" \
                       % (working_hosts_number, "\n\t\t".join(msg_texts))
            logging.info(msg_text)
            msg.send_message(msg_text)
            sleep_cnt = 0

        if result_queue.empty():
            time.sleep(sleep_time)
            sleep_cnt += 1
            continue

        whn_diff = process_result_queue(result_queue, isc_tasks)
        working_hosts_number += whn_diff[0]

    msg_texts = []
    attached_files = []
    for it in isc_tasks:
        msg_texts.append(it.task_finish())
        attached_files.append(it.result_file)

    msg_text = "isc tasks finish! \n\t\t%s" % "\n\t\t".join(msg_texts)
    logging.info(msg_text)
    msg.send_message(msg=msg_text, attached_files=attached_files)
Esempio n. 22
0
    def init_kmn_isc_task_master_from_config(
            cls, isc_config_file="isets-tasks.json", sleep_time=30):
        start_time = datetime.now()
        SearchMasterQueueManger.register("get_task_queue",
                                         callable=cls.get_global_task_queue)
        SearchMasterQueueManger.register("get_result_queue",
                                         callable=cls.get_global_result_queue)
        manager = SearchMasterQueueManger(address=(config.task_host,
                                                   config.task_host_port),
                                          authkey=bytes(config.task_host_key,
                                                        encoding="utf-8"))
        manager.start()
        task_queue = manager.get_task_queue()
        result_queue = manager.get_result_queue()
        localhost_ip = ssh.get_host_ip()

        task_generator = Pool(2)
        task_generator.apply_async(cls.itask_slices_generator,
                                   args=(cls, isc_config_file))
        task_generator.close()

        working_hosts_number = 0
        msg_text = "isc task master start, load isc tasks from %s" % (
            isc_config_file)
        logging.info(msg_text)
        msg.send_message(msg_text)

        isc_tasks_cfg = ITaskConfig(isc_config_file)
        isc_tasks = isc_tasks_cfg.isc_tasks
        for itask in isc_tasks:
            itask.init_task_numbers()

        sleep_cnt = 0
        online_hosts = set()

        progress_msg_cnt = 10
        task_finish = False
        print_loop = 100000
        print_cnt = 0
        while not task_finish:
            print_cnt += 1

            if print_cnt == print_loop:
                cls.send_itasks_progress_info(cls, isc_tasks, task_queue,
                                              working_hosts_number, False)
                sleep_cnt = 0
                print_cnt = 0

            if sleep_cnt == progress_msg_cnt:
                cls.send_itasks_progress_info(cls, isc_tasks, task_queue,
                                              working_hosts_number, False)
                sleep_cnt = 0
                print_cnt = 0

            task_finish = cls.check_itasks_status(cls, isc_tasks, online_hosts,
                                                  task_queue,
                                                  working_hosts_number)
            if result_queue.empty():
                time.sleep(sleep_time)
                sleep_cnt += 1
                continue

            whn_diff = cls.process_result_queue(cls, result_queue, isc_tasks)

            whn_number = whn_diff[0]
            host_ip = whn_diff[1]
            working_hosts_number += whn_number

            if whn_number == 1:
                if host_ip != localhost_ip:
                    online_hosts.add(host_ip)
                    cls.update_nse_files_to_new_host(host_ip, isc_tasks)
            elif whn_number == -1:
                if host_ip != localhost_ip:
                    online_hosts.remove(host_ip)

        task_generator.join()

        while working_hosts_number > 0:
            if sleep_cnt == 10:
                cls.send_itasks_progress_info(cls, isc_tasks, task_queue,
                                              working_hosts_number, True)
                sleep_cnt = 0

            if result_queue.empty():
                time.sleep(sleep_time)
                sleep_cnt += 1
                continue

            whn_diff = cls.process_result_queue(cls, result_queue, isc_tasks)
            working_hosts_number += whn_diff[0]

        msg_texts = []
        attached_files = []
        for it in isc_tasks:
            it.task_finish()
            msg_texts.append(it.get_final_detail_progress_info())
            attached_files.append(it.result_file)

        msg_text = "isc tasks finish! \n\t\t%s" % "\n\t\t".join(msg_texts)
        logging.info(msg_text)
        msg.send_message(msg=msg_text, attached_files=attached_files)
        return isc_tasks
Esempio n. 23
0
    def init_kmn_isc_task_master_from_config(
            cls, isc_config_file="isets-tasks.json", sleep_time=30):
        manager, task_queue, ht_task_queue, result_queue = \
            SearchQueueManager.init_task_master_queue_manager()
        manager_tuple = (manager, task_queue, ht_task_queue, result_queue)
        localhost_ip = ssh.get_host_ip()
        ts_generator_pool = cls.init_task_slices_generator_pool(
            cls, isc_config_file)
        ht_pool = cls.init_pre_task_worker_pool(cls, isc_config_file,
                                                result_queue)
        working_hosts_number = 0
        # ht_checking_results = list()

        isc_tasks_cfg = ITaskConfig(isc_config_file)
        isc_tasks = isc_tasks_cfg.isc_tasks
        for itask in isc_tasks:
            itask.init_task_numbers()
            if os.path.exists(itask.result_file):
                os.remove(itask.result_file)
        #     ht_checking_results.append(list())

        msg_text = "isc task master start, load %d isc tasks from %s" % (
            len(isc_tasks), isc_config_file)
        logging.info(msg_text)
        msg.send_message(msg_text)

        sleep_cnt = 0
        online_hosts = set()

        progress_msg_cnt = 10
        task_finish = False
        print_loop = 100
        print_cnt = 0

        while not task_finish:
            print_cnt += 1

            if print_cnt == print_loop:
                cls.send_itasks_progress_info(cls, isc_tasks, manager_tuple,
                                              working_hosts_number, False)
                sleep_cnt = 0
                print_cnt = 0

            if sleep_cnt == progress_msg_cnt:
                cls.send_itasks_progress_info(cls, isc_tasks, manager_tuple,
                                              working_hosts_number, False)
                sleep_cnt = 0
                print_cnt = 0

            task_finish = cls.check_itasks_status(cls, isc_tasks, online_hosts,
                                                  manager_tuple,
                                                  working_hosts_number)
            if result_queue.empty():
                time.sleep(sleep_time)
                sleep_cnt += 1
                continue

            whn_diff = cls.process_result_queue(cls, result_queue, isc_tasks)

            whn_number = whn_diff[0]
            host_ip = whn_diff[1]
            working_hosts_number += whn_number

            if whn_number == 1:
                if host_ip != localhost_ip:
                    online_hosts.add(host_ip)
            elif whn_number == -1:
                if host_ip != localhost_ip:
                    online_hosts.remove(host_ip)

        ts_generator_pool.join()
        ht_pool.join()
        HTCheckingWorker.send_worker_terminate_info(HTCheckingWorker,
                                                    localhost_ip, result_queue)

        while working_hosts_number > 0:
            if sleep_cnt == 10:
                cls.send_itasks_progress_info(cls, isc_tasks, manager_tuple,
                                              working_hosts_number, True)
                sleep_cnt = 0

            if result_queue.empty():
                time.sleep(sleep_time)
                sleep_cnt += 1
                continue

            whn_diff = cls.process_result_queue(cls, result_queue, isc_tasks)
            working_hosts_number += whn_diff[0]

        msg_text = "isc tasks finish!"
        logging.info(msg_text)
        msg.send_message(msg=msg_text)

        for it in isc_tasks:
            it.dump_tmp_se_condition_saving_mem()
            msg_text = it.get_final_detail_progress_info()
            logging.info(msg_text)
            msg.send_message(msg=msg_text)

        return isc_tasks
    def itask_slices_generator(cls, isc_config_file="isets-tasks.json"):
        msg_text = "%s init task slices generator ..." % str(cls)
        logging.info(msg_text)
        msg.send_message(msg_text)

        SearchWorkerQueueManger.register("get_task_queue")
        SearchWorkerQueueManger.register("get_result_queue")
        manager = SearchWorkerQueueManger(address=(config.task_host,
                                                   config.task_host_port),
                                          authkey=bytes(config.task_host_key,
                                                        encoding="utf-8"))
        manager.connect()
        task_queue = manager.get_task_queue()
        result_queue = manager.get_result_queue()

        isc_tasks_cfg = ITaskConfig(isc_config_file)
        isc_tasks = isc_tasks_cfg.isc_tasks

        for tid in range(len(isc_tasks)):
            it = isc_tasks[tid]
            min_ne = it.min_ne
            max_ne = it.max_ne
            unknown_iset_number = len(it.meta_data.search_space_iset_ids)
            rule_number = it.rule_number

            left_zone_iset_ids = it.meta_data.search_i4_composed_iset_ids

            left_zone_length = len(left_zone_iset_ids)
            right_zone_length = unknown_iset_number - left_zone_length
            is_use_extended_rules = it.is_use_extended_rules

            for i in range(min_ne, max_ne + 1):
                ne_iset_number = i
                for left_iset_number in range(ne_iset_number + 1):
                    right_iset_number = ne_iset_number - left_iset_number
                    if left_iset_number > left_zone_length or right_iset_number > right_zone_length:
                        continue

                    task_iter = itertools.combinations(left_zone_iset_ids,
                                                       left_iset_number)
                    for left_ti in task_iter:
                        left_iset_ids = list(left_ti)
                        is_contain_semi_valid_rule = iscm.check_contain_rules_without_i_n_iset(
                            4, left_iset_ids, rule_number,
                            is_use_extended_rules)

                        if is_contain_semi_valid_rule:
                            check_cnt = 0
                            # C(right_zone_length, right_iset_number)
                            task_number = CombinaryCounter.compute_comb(
                                right_zone_length, right_iset_number)
                            semi_valid_skip_cnt = task_number
                            stat_item = (ITaskSignal.stat_signal, tid,
                                         ne_iset_number, check_cnt,
                                         task_number, semi_valid_skip_cnt,
                                         None)
                            result_queue.put(stat_item)
                        else:
                            task_item = (tid, (ne_iset_number,
                                               set(left_zone_iset_ids),
                                               left_iset_ids))
                            # print(task_item)
                            task_queue.put(task_item)

        working_hosts_number = 5
        for i in range(working_hosts_number * 200):
            task_queue.put((ITaskSignal.kill_signal, -1))
        logging.info("all itasks has been dispatched")
Esempio n. 25
0
    def itask_slice_generator_by_i4_meta(ne_iset_number, itask_id, itask,
                                         max_space_size, manager_tuple):
        task_queue = manager_tuple[1]
        result_queue = manager_tuple[3]

        kmn_key = I4RawSearchMaster.get_kmn_meta_key(*itask.k_m_n)
        i4_meta = I4RawSearchMaster.i4_meta[kmn_key]

        left_zone_length = len(itask.meta_data.search_i4_composed_iset_ids)
        search_isets_length = len(itask.meta_data.search_space_iset_ids)
        right_zone_length = search_isets_length - left_zone_length

        task_slice_cnt = 0
        if ne_iset_number <= right_zone_length:
            semi_valid_i4_slices_size = CombinaryCounter.compute_comb(
                right_zone_length, ne_iset_number)
            valid_skip_number = CombinaryCounter.compute_comb(
                right_zone_length, ne_iset_number)
            result_tuple = (ITaskSignal.stat_signal, itask_id, ne_iset_number,
                            0, valid_skip_number, valid_skip_number, None)
            result_queue.put(result_tuple)

        for left_choice in range(1, left_zone_length + 1):
            right_choice = ne_iset_number - left_choice
            if right_choice > right_zone_length or left_choice > ne_iset_number:
                continue

            single_slice_right_task_number = CombinaryCounter.compute_comb(
                right_zone_length, right_choice)
            task_i4_slice_number = max_space_size // single_slice_right_task_number + 1
            non_semi_valid_i4_slices_size = i4_meta[left_choice]
            itask_sizes = non_semi_valid_i4_slices_size // task_i4_slice_number

            if itask_sizes < 92 or itask_sizes > 1200:
                itask_sizes = 1200
                if itask_sizes > non_semi_valid_i4_slices_size:
                    itask_sizes = non_semi_valid_i4_slices_size

            task_i4_slice_number = non_semi_valid_i4_slices_size // itask_sizes

            itask_splitting_points = [
                i * task_i4_slice_number for i in range(itask_sizes)
            ]
            if len(itask_splitting_points) == 0:
                itask_splitting_points.append(0)

            if itask_splitting_points[-1] < non_semi_valid_i4_slices_size:
                itask_splitting_points.append(non_semi_valid_i4_slices_size)

            for i in range(1, len(itask_splitting_points)):
                itask_slice_tuple = (left_choice,
                                     itask_splitting_points[i - 1],
                                     itask_splitting_points[i], right_choice)
                itask_slice_tuple = (itask_id, itask_slice_tuple)
                task_queue.put(itask_slice_tuple)
                task_slice_cnt += 1

            total_i4_silces_size = CombinaryCounter.compute_comb(
                left_zone_length, left_choice)
            semi_valid_i4_slices_size = total_i4_silces_size - non_semi_valid_i4_slices_size
            if semi_valid_i4_slices_size > 0:
                valid_skip_number = semi_valid_i4_slices_size * single_slice_right_task_number
                result_tuple = (ITaskSignal.stat_signal, itask_id,
                                ne_iset_number, 0, valid_skip_number,
                                valid_skip_number, None)
                result_queue.put(result_tuple)

        msg_text = "itask %d-%d-%d ne iset number %d, put %d task slices" % (
            *itask.k_m_n, ne_iset_number, task_slice_cnt)
        logging.info(msg_text)
        msg.send_message(msg_text)