def process_result_queue(result_queue, isc_tasks):
working_hosts_number_diff = (0, 0)
result = result_queue.get()
result_state = result[0]
isc_task_id = result[1]
if result_state == kill_signal:
host_ip = result[2]
working_hosts_number_diff = (-1, host_ip)
msg_text = "task host %s:%s exit" % (isc_task_id, host_ip)
logging.info(msg_text)
msg.send_message(msg_text)
elif result_state == add_worker_signal:
host_ip = result[2]
msg_text = "task host %s:%s is online" % (isc_task_id, host_ip)
working_hosts_number_diff = (1, host_ip)
logging.info(msg_text)
msg.send_message(msg_text)
elif result_state == stat_signal:
ne_iset_number = result[2]
task_complete_number = result[3]
task_running_time = result[4]
isc_tasks[isc_task_id].set_task_complete_number(task_complete_number, ne_iset_number)
isc_tasks[isc_task_id].set_task_running_time(task_running_time)
elif result_state == se_condition_signal:
iconditions = result[2]
for ic in iconditions:
isc_tasks[isc_task_id].insert_se_condition(ic)
# put_isc_task_items(isc_task_id, itask_items, msg_text, task_queue)
elif result_state == nse_condition_signal:
iconditions = result[2]
for ic in iconditions:
isc_tasks[isc_task_id].insert_nse_condition(ic)
return working_hosts_number_diff
def process_result_queue(task_queue, result_queue, isc_tasks):
working_hosts_number_diff = 0
result = result_queue.get()
result_state = result[0]
isc_task_id = result[1]
if result_state == kill_signal:
working_hosts_number_diff = -1
msg_text = "task host %s exit" % isc_task_id
logging.info(msg_text)
msg.send_message(msg_text)
elif result_state == add_worker_signal:
msg_text = "task host %s is online" % isc_task_id
working_hosts_number_diff = 1
logging.info(msg_text)
msg.send_message(msg_text)
elif result_state == stat_signal:
ne_iset_number = result[2]
task_complete_number = result[3]
task_running_time = result[4]
isc_tasks[isc_task_id].set_task_complete_number(
task_complete_number, ne_iset_number)
isc_tasks[isc_task_id].set_task_running_time(task_running_time)
elif result_state == condition_signal:
iconditions = result[2]
for ic in iconditions:
if ic.contain_se_valid_rules:
itask_items, msg_text = isc_tasks[
isc_task_id].generate_isc_task_items_by_base_icondition(ic)
else:
itask_items, msg_text = isc_tasks[
isc_task_id].insert_se_condition(ic)
put_isc_task_items(isc_task_id, itask_items, msg_text, task_queue)
return working_hosts_number_diff
def check_itasks_status(itasks, task_queue, working_hosts_number, is_frequent_log):
is_finish = True
is_task_queue_change = False
for tid in range(len(itasks)):
it = itasks[tid]
if not it.is_task_finish:
current_ne_number = it.working_ne_iset_numbers
task_complete = it.incremental_task_complete_number[current_ne_number]
task_total = it.incremental_task_number[current_ne_number]
if task_complete == task_total:
if it.is_early_terminate():
continue
if current_ne_number < it.max_ne:
is_task_queue_change = True
is_finish = False
it.flush_non_se_condition()
task_slices, msg_text = it.get_check_itasks_by_non_empty_iset_number()
for ts in task_slices:
task_queue.put((tid, ts))
logging.info(msg_text)
msg.send_message(msg_text)
else:
it.is_task_finish = True
else:
is_finish = False
if is_task_queue_change and is_frequent_log:
send_itasks_progress_info(itasks, task_queue, working_hosts_number)
return is_finish
def itask_slices_generator(cls, isc_config_file):
msg_text = "%s init task slices generator ..." % str(cls)
logging.info(msg_text)
msg.send_message(msg_text)
manager, task_queue, ht_task_queue, result_queue = \
SearchQueueManager.init_task_worker_queue_manager()
isc_tasks_cfg = ITaskConfig(isc_config_file)
isc_tasks = isc_tasks_cfg.isc_tasks
for tid in range(len(isc_tasks)):
it = isc_tasks[tid]
ht_task_number = it.ht_task_number
batch_size = 100
ht_task_slices = [
i for i in range(0, ht_task_number + 1, batch_size)
]
if ht_task_slices[-1] < ht_task_number:
ht_task_slices.append(ht_task_number)
for i in range(1, len(ht_task_slices)):
ts = (tid, (ht_task_slices[i - 1], ht_task_slices[i]))
ht_task_queue.put(ts)
working_hosts_number = 5
for i in range(working_hosts_number * 200):
ht_task_queue.put((ITaskSignal.kill_signal, -1))
logging.info("all itasks has been dispatched")
msg.send_message("all itasks has been dispatched")
def insert_found_conditions(itask, iconditions, is_se_condition=True):
if is_se_condition:
for ic in iconditions:
itask.insert_se_condition(ic)
else:
for ic in iconditions:
msg_text = "%d-%d-%d nse condition: %s" % (*itask.k_m_n,
str(ic))
logging.error(msg_text)
msg.send_message(msg_text)
def process_working_host_change(info, is_add=True):
host_name = info[1]
host_ip = info[2]
if is_add:
diff = 1
msg_text = "task host %s:%s is online" % (host_name, host_ip)
else:
diff = -1
msg_text = "task host %s:%s exit" % (host_name, host_ip)
working_hosts_number_diff = (diff, host_ip)
logging.info(msg_text)
msg.send_message(msg_text)
return working_hosts_number_diff
def itask_slices_generator(cls, isc_config_file="isets-tasks.json"):
msg_text = "%s init task slices generator ..." % str(cls)
logging.info(msg_text)
msg.send_message(msg_text)
SearchWorkerQueueManger.register("get_task_queue")
SearchWorkerQueueManger.register("get_result_queue")
manager = SearchWorkerQueueManger(address=(config.task_host,
config.task_host_port),
authkey=bytes(config.task_host_key,
encoding="utf-8"))
manager.connect()
task_queue = manager.get_task_queue()
isc_tasks_cfg = ITaskConfig(isc_config_file)
isc_tasks = isc_tasks_cfg.isc_tasks
for tid in range(len(isc_tasks)):
it = isc_tasks[tid]
min_ne = it.min_ne
max_ne = it.max_ne
search_iset_ids = it.meta_data.search_space_iset_ids
unknown_iset_number = len(search_iset_ids)
left_length = int(unknown_iset_number / 2)
if left_length > 12:
left_length = 12
right_length = unknown_iset_number - left_length
left_zone_isets = search_iset_ids[0:left_length]
for i in range(min_ne, max_ne + 1):
ne_iset_number = i
for left_iset_number in range(ne_iset_number + 1):
right_iset_number = ne_iset_number - left_iset_number
if left_iset_number > left_length or right_iset_number > right_length:
continue
task_iter = itertools.combinations(left_zone_isets,
left_iset_number)
for left_ti in task_iter:
task_item = (tid,
(ne_iset_number, set(left_zone_isets),
list(left_ti)))
# print(task_item)
task_queue.put(task_item)
working_hosts_number = 5
for i in range(working_hosts_number * 200):
task_queue.put((ITaskSignal.kill_signal, -1))
logging.info("all itasks has been dispatched")
def send_itasks_progress_info(cls,
isc_tasks,
task_queue,
working_hosts_number,
is_all_task_dispatched=False):
msg_texts = cls.dump_isc_task_results(isc_tasks)
if is_all_task_dispatched:
msg_text = "all isc tasks are discatched, DO NOT add new worker! isc tasks progress info, remain %d task hosts: \n\t\t%s" \
% (working_hosts_number, "\n\t\t".join(msg_texts))
else:
msg_text = "isc tasks progress info, remain %d task hosts, %d task slices: \n\t\t%s" % (
working_hosts_number, task_queue.qsize(),
"\n\t\t".join(msg_texts))
logging.info(msg_text)
msg.send_message(msg_text)
def itask_slices_generator(cls, isc_config_file):
msg_text = "%s init task slices generator ..." % str(cls)
logging.info(msg_text)
msg.send_message(msg_text)
manager, task_queue, ht_task_queue, result_queue = \
SearchQueueManager.init_task_worker_queue_manager()
isc_tasks_cfg = ITaskConfig(isc_config_file)
isc_tasks = isc_tasks_cfg.isc_tasks
for tid in range(len(isc_tasks)):
it = isc_tasks[tid]
min_ne = it.min_ne
max_ne = it.max_ne
right_zone_iset_ids = set(
copy.deepcopy(it.meta_data.search_space_iset_ids))
left_zone_iset_ids = set(it.meta_data.search_i4_composed_iset_ids)
max_left_zone_length = 12
if len(left_zone_iset_ids) > max_left_zone_length:
left_zone_iset_ids = list(
left_zone_iset_ids)[0:max_left_zone_length]
left_zone_iset_ids = set(left_zone_iset_ids)
right_zone_iset_ids = right_zone_iset_ids.difference(
left_zone_iset_ids)
for ne_iset_number in range(min_ne, max_ne + 1):
if not cls.check_itask_terminate_status(it):
task_slices = CombinationSearchingSpaceSplitter.near_uniform_vandermonde_generator(
left_zone_iset_ids, right_zone_iset_ids,
ne_iset_number)
ts_cnt = 0
for ts in task_slices:
new_ts = (set(ts[0]), set(ts[1]), ts[2])
task_queue.put((tid, new_ts))
ts_cnt += 1
if ts_cnt % 10000 == 0 and cls.check_itask_terminate_status(
it):
break
working_hosts_number = 5
for i in range(working_hosts_number * 200):
task_queue.put((ITaskSignal.kill_signal, -1))
logging.info("all itasks has been dispatched")
def itask_slices_generator(cls, isc_config_file):
max_space_size = 10000000000
msg_text = "%s init task slices generator ..." % str(cls)
logging.info(msg_text)
msg.send_message(msg_text)
manager_tuple = SearchQueueManager.init_task_worker_queue_manager()
task_queue = manager_tuple[1]
isc_tasks_cfg = ITaskConfig(isc_config_file)
isc_tasks = isc_tasks_cfg.isc_tasks
for tid in range(len(isc_tasks)):
it = isc_tasks[tid]
min_ne = it.min_ne
max_ne = it.max_ne
rule_number = sum(it.k_m_n)
isnse.clear_task_space_layer_finish_flag_files(
*it.k_m_n, min_ne, max_ne)
for ne_iset_number in range(min_ne, max_ne + 1):
msg_text = "generating %d-%d-%d %d layer task slices" % (
*it.k_m_n, ne_iset_number)
logging.info(msg_text)
cls.itask_slice_generator_by_i4_meta(ne_iset_number, tid, it,
max_space_size,
manager_tuple)
# if ne_iset_number <= rule_number:
# cls.itask_slice_generator_by_i4_meta(ne_iset_number, tid, it, max_space_size, manager_tuple)
# else:
# if not cls.check_itask_terminate_status(it):
# flag_file = isnse.get_task_space_layer_finish_flag_file(*it.k_m_n, ne_iset_number - 2)
# while not pathlib.Path(flag_file).exists():
# if cls.check_itask_terminate_status(it):
# break
# time.sleep(1)
#
# cls.itask_slice_generator_by_i4_meta(ne_iset_number, tid, it, max_space_size, manager_tuple)
working_hosts_number = 5
for i in range(working_hosts_number * 200):
task_queue.put((ITaskSignal.kill_signal, -1))
logging.info("all itasks has been dispatched")
def send_itasks_progress_info(cls,
isc_tasks,
manager_tuple,
working_hosts_number,
is_all_task_dispatched=False):
task_queue = manager_tuple[1]
ht_task_queue = manager_tuple[2]
result_queue = manager_tuple[3]
msg_texts = cls.dump_isc_task_results(isc_tasks)
msg_text = "isc tasks progress info, remain %d task hosts, %d task slices, %d ht task slices, %d results items: \n\t\t%s" % (
working_hosts_number, task_queue.qsize(), ht_task_queue.qsize(),
result_queue.qsize(), "\n\t\t".join(msg_texts))
if is_all_task_dispatched:
msg_text = "all isc tasks are discatched, DO NOT add new worker! " + msg_text
logging.info(msg_text)
msg.send_message(msg_text)
def process_result_queue(result_queue, isc_tasks):
working_hosts_number_diff = 0
result = result_queue.get()
result_state = result[0]
isc_task_id = result[1]
if result_state == kill_signal:
working_hosts_number_diff = -1
msg_text = "task host %s exit" % isc_task_id
logging.info(msg_text)
msg.send_message(msg_text)
elif result_state == add_worker_signal:
msg_text = "task host %s is online" % isc_task_id
working_hosts_number_diff = 1
logging.info(msg_text)
msg.send_message(msg_text)
elif result_state == stat_signal:
isc_tasks[isc_task_id].task_complete_number += result[2]
isc_tasks[isc_task_id].set_task_running_time(result[3])
elif result_state == condition_signal:
isc_tasks[isc_task_id].insert_se_condition(result[2])
return working_hosts_number_diff
def itask_slices_generator(cls, isc_config_file):
msg_text = "%s init task slices generator ..." % str(cls)
logging.info(msg_text)
msg.send_message(msg_text)
manager, task_queue, ht_task_queue, result_queue = \
SearchQueueManager.init_task_worker_queue_manager()
isc_tasks_cfg = ITaskConfig(isc_config_file)
isc_tasks = isc_tasks_cfg.isc_tasks
for tid in range(len(isc_tasks)):
it = isc_tasks[tid]
i4_iset_ids = list(it.meta_data.search_i4_composed_iset_ids)
i4_iset_size = len(i4_iset_ids)
min_ne = 1
max_ne = len(i4_iset_ids)
max_left_zone_length = 10
left_zone_size = max_left_zone_length
if i4_iset_size < left_zone_size:
left_zone_size = i4_iset_size // 2
left_zone_iset_ids = i4_iset_ids[0:left_zone_size]
right_zone_iset_ids = i4_iset_ids[left_zone_size:]
for ne_iset_number in range(min_ne, max_ne + 1):
task_slices = CombinationSearchingSpaceSplitter.vandermonde_generator(
left_zone_iset_ids, right_zone_iset_ids, ne_iset_number)
for ts in task_slices:
task_queue.put((tid, ts))
working_hosts_number = 5
for i in range(working_hosts_number * 200):
task_queue.put((ITaskSignal.kill_signal, -1))
logging.info("all itasks has been dispatched")
def send_itasks_progress_info(cls,
result_records,
manager_tuple,
working_hosts_number,
is_all_task_dispatched=False):
task_queue = manager_tuple[1]
ht_task_queue = manager_tuple[2]
result_queue = manager_tuple[3]
msg_texts = list()
for i in range(len(result_records)):
msg_text = "task %d has %d tasks, complete %d tasks" % (
i, result_records[i][0], result_records[i][1])
msg_texts.append(msg_text)
msg_text = "isc tasks progress info, remain %d task hosts, %d task slices, %d ht task slices, %d results items: \n\t\t%s" % (
working_hosts_number, task_queue.qsize(), ht_task_queue.qsize(),
result_queue.qsize(), "\n\t\t".join(msg_texts))
if is_all_task_dispatched:
msg_text = "all isc tasks are discatched, DO NOT add new worker! " + msg_text
logging.info(msg_text)
msg.send_message(msg_text)
def send_itasks_progress_info(isc_tasks, task_queue, working_hosts_number):
msg_texts = dump_isc_task_results(isc_tasks)
msg_text = "isc tasks progress info, remain %d task hosts, %d task slices: \n\t\t%s" % (
working_hosts_number, task_queue.qsize(), "\n\t\t".join(msg_texts))
logging.info(msg_text)
msg.send_message(msg_text)
def init_kmn_isc_task_master_from_config(
cls, isc_config_file="isets-tasks.json", sleep_time=30):
manager, task_queue, ht_task_queue, result_queue = \
SearchQueueManager.init_task_master_queue_manager()
manager_tuple = (manager, task_queue, ht_task_queue, result_queue)
localhost_ip = ssh.get_host_ip()
isc_tasks_cfg = ITaskConfig(isc_config_file)
isc_tasks = isc_tasks_cfg.isc_tasks
result_record = list()
for itask in isc_tasks:
isnse.clear_task_terminate_flag_files(*itask.k_m_n)
i4_iset_size = len(itask.meta_data.search_i4_composed_iset_ids)
file = i4u.get_kmn_i4_all_result_file(*itask.k_m_n)
if os.path.exists(file):
os.remove(file)
record = [2**i4_iset_size - 1, 0, list(), file]
result_record.append(record)
ts_generator_pool = cls.init_task_slices_generator_pool(
cls, isc_config_file)
pre_task_pool = cls.init_pre_task_worker_pool(cls, isc_config_file,
result_queue)
working_hosts_number = 0
msg_text = "isc task master start, load %d isc tasks from %s" % (
len(isc_tasks), isc_config_file)
logging.info(msg_text)
msg.send_message(msg_text)
sleep_cnt = 0
online_hosts = set()
progress_msg_cnt = 10
task_finish = False
print_loop = 10
print_cnt = 0
while not task_finish:
print_cnt += 1
if print_cnt == print_loop:
cls.send_itasks_progress_info(cls, result_record,
manager_tuple,
working_hosts_number, False)
sleep_cnt = 0
print_cnt = 0
if sleep_cnt == progress_msg_cnt:
cls.send_itasks_progress_info(cls, result_record,
manager_tuple,
working_hosts_number, False)
sleep_cnt = 0
print_cnt = 0
task_finish = cls.check_i4_tasks_status(cls, result_record)
if result_queue.empty():
time.sleep(sleep_time)
sleep_cnt += 1
continue
whn_diff = cls.process_i4_result_queue(cls, result_queue,
result_record)
whn_number = whn_diff[0]
host_ip = whn_diff[1]
working_hosts_number += whn_number
if whn_number == 1:
if host_ip != localhost_ip:
online_hosts.add(host_ip)
elif whn_number == -1:
if host_ip != localhost_ip:
online_hosts.remove(host_ip)
ts_generator_pool.join()
pre_task_pool.join()
I4SearchWorker.send_worker_terminate_info(I4SearchWorker, localhost_ip,
result_queue)
while working_hosts_number > 0:
if sleep_cnt == 10:
cls.send_itasks_progress_info(cls, result_record,
manager_tuple,
working_hosts_number, True)
sleep_cnt = 0
if result_queue.empty():
time.sleep(sleep_time)
sleep_cnt += 1
continue
whn_diff = cls.process_i4_result_queue(cls, result_queue,
result_record)
working_hosts_number += whn_diff[0]
msg_text = "isc tasks finish!"
logging.info(msg_text)
msg.send_message(msg=msg_text)
cls.send_itasks_progress_info(cls, result_record, manager_tuple,
working_hosts_number, True)
return isc_tasks
def init_kmn_isc_task_master_from_config(isc_config_file="isets-tasks.json", sleep_time=60, is_use_extended_rules=True):
start_time = datetime.now()
ISCFileTaskMasterQueueManager.register("get_task_queue", callable=get_task_queue)
ISCFileTaskMasterQueueManager.register("get_result_queue", callable=get_result_queue)
manager = ISCFileTaskMasterQueueManager(address=(config.task_host, config.task_host_port),
authkey=bytes(config.task_host_key, encoding="utf-8"))
manager.start()
task_queue = manager.get_task_queue()
result_queue = manager.get_result_queue()
working_hosts_number = 0
msg_text = "isc task master start, load isc tasks from %s" % (isc_config_file)
logging.info(msg_text)
msg.send_message(msg_text)
isc_tasks_cfg = isc_cfg.ISCTaskConfig(isc_config_file, is_use_extended_rules)
isc_tasks = isc_tasks_cfg.isc_tasks
for isc_id in range(len(isc_tasks)):
isc_task_slices = isc_tasks[isc_id]
isc_task_items = isc_task_slices.load_isc_task_items()
for it in isc_task_items:
task_queue.put((isc_id, it))
msg_text = isc_task_slices.get_isc_task_load_message()
logging.info(msg_text)
msg.send_message(msg_text)
sleep_cnt = 0
while not task_queue.empty():
if sleep_cnt == 10:
msg_texts = dump_isc_task_results(isc_tasks)
msg_text = "isc tasks progress info, remain %d task hosts, %d task slices: \n\t\t%s" % (
working_hosts_number, task_queue.qsize(), "\n\t\t".join(msg_texts))
logging.info(msg_text)
msg.send_message(msg_text)
sleep_cnt = 0
if result_queue.empty():
time.sleep(sleep_time)
sleep_cnt += 1
continue
whn_diff = process_result_queue(result_queue, isc_tasks)
working_hosts_number += whn_diff
msg_text = "all isc task slices are discatched!"
logging.info(msg_text)
msg.send_message(msg_text)
for i in range(working_hosts_number * 200):
task_queue.put((kill_signal, -1))
while working_hosts_number > 0:
if sleep_cnt == 10:
msg_texts = dump_isc_task_results(isc_tasks)
msg_text = "all isc tasks are discatched, DO NOT add new worker! isc tasks progress info, remain %d task hosts: \n\t\t%s" \
% (working_hosts_number, "\n\t\t".join(msg_texts))
logging.info(msg_text)
msg.send_message(msg_text)
sleep_cnt = 0
if result_queue.empty():
time.sleep(sleep_time)
sleep_cnt += 1
continue
whn_diff = process_result_queue(result_queue, isc_tasks)
working_hosts_number += whn_diff
msg_texts = []
attached_files = []
for it in isc_tasks:
msg_texts.append(it.task_finish())
attached_files.append(it.result_file)
msg_text = "isc tasks finish! \n\t\t%s" % "\n\t\t".join(msg_texts)
logging.info(msg_text)
msg.send_message(msg=msg_text, attached_files=attached_files)
def itask_slices_generator(cls, isc_config_file):
max_space_size = 100000000000
msg_text = "%s init task slices generator ..." % str(cls)
logging.info(msg_text)
msg.send_message(msg_text)
manager, task_queue, ht_task_queue, result_queue = \
SearchQueueManager.init_task_worker_queue_manager()
isc_tasks_cfg = ITaskConfig(isc_config_file)
isc_tasks = isc_tasks_cfg.isc_tasks
for tid in range(len(isc_tasks)):
it = isc_tasks[tid]
min_ne = it.min_ne
max_ne = it.max_ne
isnse.clear_task_space_layer_finish_flag_files(*it.k_m_n, min_ne, max_ne)
left_zone_length = len(it.meta_data.search_i4_composed_iset_ids)
search_isets = copy.deepcopy(it.meta_data.search_space_iset_ids)
search_isets_length = len(search_isets)
max_left_zone_length = 12
if left_zone_length > max_left_zone_length:
left_zone_length = 12
rule_number = sum(it.k_m_n)
left_zone_iset_ids = search_isets[0:left_zone_length]
right_zone_iset_ids = search_isets[left_zone_length:]
for ne_iset_number in range(min_ne, max_ne + 1):
msg_text = "generating %d-%d-%d %d layer task slices" % (*it.k_m_n, ne_iset_number)
logging.info(msg_text)
if ne_iset_number <= rule_number:
left_split = True
task_slices = CombinationSearchingSpaceSplitter.vandermonde_generator(
left_zone_iset_ids, right_zone_iset_ids, ne_iset_number)
for ts in task_slices:
new_ts = (left_split, set(ts[0]), left_zone_length, ts[2])
task_queue.put((tid, new_ts))
else:
if not cls.check_itask_terminate_status(it):
flag_file = isnse.get_task_space_layer_finish_flag_file(*it.k_m_n, ne_iset_number - 2)
while not pathlib.Path(flag_file).exists():
if cls.check_itask_terminate_status(it):
break
time.sleep(1)
task_slices = CombinationSearchingSpaceSplitter.merge_small_near_uniform_vandermonde_generator(
left_zone_iset_ids, right_zone_iset_ids, ne_iset_number, max_space_size=max_space_size)
ts_cnt = 0
for ts in task_slices:
task_queue.put((tid, ts))
ts_cnt += 1
if ts_cnt % 10000 == 0 and cls.check_itask_terminate_status(it):
break
working_hosts_number = 5
for i in range(working_hosts_number * 200):
task_queue.put((ITaskSignal.kill_signal, -1))
logging.info("all itasks has been dispatched")
def init_kmn_isc_task_master_from_config(cls, isc_config_file="isets-tasks.json", sleep_time=30):
start_time = datetime.now()
manager, task_queue, ht_task_queue, result_queue = \
SearchQueueManager.init_task_master_queue_manager()
manager_tuple = (manager, task_queue, ht_task_queue, result_queue)
localhost_ip = ssh.get_host_ip()
isc_tasks_cfg = ITaskConfig(isc_config_file)
isc_tasks = isc_tasks_cfg.isc_tasks
for itask in isc_tasks:
itask.init_task_numbers()
isnse.clear_task_terminate_flag_files(*itask.k_m_n)
ts_generator_pool = cls.init_task_slices_generator_pool(cls, isc_config_file)
pre_pool = cls.init_pre_task_worker_pool(cls, isc_config_file, result_queue)
working_hosts_number = 0
msg_text = "isc task master start, load %d isc tasks from %s" % (len(isc_tasks), isc_config_file)
logging.info(msg_text)
msg.send_message(msg_text)
sleep_cnt = 0
online_hosts = set()
progress_msg_cnt = 10
task_finish = False
print_loop = 100
print_cnt = 0
while not task_finish:
print_cnt += 1
if print_cnt == print_loop:
cls.send_itasks_progress_info(cls, isc_tasks, manager_tuple, working_hosts_number, False)
for it in isc_tasks:
it.save_progress_info()
sleep_cnt = 0
print_cnt = 0
if sleep_cnt == progress_msg_cnt:
cls.send_itasks_progress_info(cls, isc_tasks, manager_tuple, working_hosts_number, False)
for it in isc_tasks:
it.save_progress_info()
sleep_cnt = 0
print_cnt = 0
task_finish = cls.check_itasks_status(cls, isc_tasks, online_hosts, manager_tuple, working_hosts_number)
if result_queue.empty():
time.sleep(sleep_time)
sleep_cnt += 1
continue
whn_diff = cls.process_result_queue(cls, result_queue, isc_tasks)
whn_number = whn_diff[0]
host_ip = whn_diff[1]
working_hosts_number += whn_number
if whn_number == 1:
if host_ip != localhost_ip:
online_hosts.add(host_ip)
cls.update_nse_files_to_new_host(host_ip, isc_tasks)
elif whn_number == -1:
if host_ip != localhost_ip:
online_hosts.remove(host_ip)
ts_generator_pool.join()
pre_pool.join()
RawIConditionSearchWorker.send_worker_terminate_info(RawIConditionSearchWorker, localhost_ip, result_queue)
while working_hosts_number > 0:
if sleep_cnt == 10:
cls.send_itasks_progress_info(cls, isc_tasks, manager_tuple, working_hosts_number, True)
sleep_cnt = 0
if result_queue.empty():
time.sleep(sleep_time)
sleep_cnt += 1
continue
whn_diff = cls.process_result_queue(cls, result_queue, isc_tasks)
working_hosts_number += whn_diff[0]
end_time = datetime.now()
msg_text = "isc tasks finish, running time: %s" % str(end_time - start_time)
logging.info(msg_text)
msg.send_message(msg=msg_text)
for it in isc_tasks:
it.task_finish()
msg_text = it.get_final_detail_progress_info()
logging.info(msg_text)
msg.send_message(msg=msg_text)
# attached_files.append(it.result_file)
return isc_tasks
def init_kmn_isc_task_master_from_config(isc_config_file="isets-tasks.json",
sleep_time=30,
is_use_extended_rules=True,
is_frequent_log=False):
start_time = datetime.now()
ISCFileTaskTerminationMasterQueueManager.register("get_task_queue",
callable=get_task_queue)
ISCFileTaskTerminationMasterQueueManager.register(
"get_result_queue", callable=get_result_queue)
manager = ISCFileTaskTerminationMasterQueueManager(
address=(config.task_host, config.task_host_port),
authkey=bytes(config.task_host_key, encoding="utf-8"))
manager.start()
task_queue = manager.get_task_queue()
result_queue = manager.get_result_queue()
localhost_ip = ssh.get_host_ip()
task_generator = Pool(2)
task_generator.apply_async(itask_slices_generator,
args=(isc_config_file, is_use_extended_rules))
task_generator.close()
working_hosts_number = 0
msg_text = "isc task master start, load isc tasks from %s" % (
isc_config_file)
logging.info(msg_text)
msg.send_message(msg_text)
isc_tasks_cfg = ISCTaskConfig(isc_config_file, is_use_extended_rules)
isc_tasks = isc_tasks_cfg.isc_tasks
for itask in isc_tasks:
itask.init_task_numbers()
sleep_cnt = 0
online_hosts = set()
progress_msg_cnt = 10
task_finish = False
print_loop = 100000
print_cnt = 0
while not task_finish:
print_cnt += 1
if print_cnt == print_loop:
send_itasks_progress_info(isc_tasks, task_queue,
working_hosts_number)
sleep_cnt = 0
print_cnt = 0
if sleep_cnt == progress_msg_cnt:
send_itasks_progress_info(isc_tasks, task_queue,
working_hosts_number)
sleep_cnt = 0
print_cnt = 0
task_finish = check_itasks_status(isc_tasks, online_hosts, task_queue,
working_hosts_number)
if result_queue.empty():
time.sleep(sleep_time)
sleep_cnt += 1
continue
whn_diff = process_result_queue(result_queue, isc_tasks)
whn_number = whn_diff[0]
host_ip = whn_diff[1]
working_hosts_number += whn_number
if whn_number == 1:
if host_ip != localhost_ip:
online_hosts.add(host_ip)
update_nse_files_to_new_host(host_ip, isc_tasks)
elif whn_number == -1:
if host_ip != localhost_ip:
online_hosts.remove(host_ip)
msg_text = "all isc task slices are discatched!"
logging.info(msg_text)
msg.send_message(msg_text)
task_generator.join()
while working_hosts_number > 0:
if sleep_cnt == 10:
msg_texts = dump_isc_task_results(isc_tasks)
msg_text = "all isc tasks are discatched, DO NOT add new worker! isc tasks progress info, remain %d task hosts: \n\t\t%s" \
% (working_hosts_number, "\n\t\t".join(msg_texts))
logging.info(msg_text)
msg.send_message(msg_text)
sleep_cnt = 0
if result_queue.empty():
time.sleep(sleep_time)
sleep_cnt += 1
continue
whn_diff = process_result_queue(result_queue, isc_tasks)
working_hosts_number += whn_diff[0]
msg_texts = []
attached_files = []
for it in isc_tasks:
it.task_finish()
msg_texts.append(it.get_final_detail_progress_info())
attached_files.append(it.result_file)
msg_text = "isc tasks finish! \n\t\t%s" % "\n\t\t".join(msg_texts)
logging.info(msg_text)
msg.send_message(msg=msg_text, attached_files=attached_files)
return isc_tasks
def init_kmn_isc_task_master_from_config(isc_config_file="isets-tasks.json", sleep_time=30, is_use_extended_rules=True, is_frequent_log=False):
start_time = datetime.now()
ISCFileTaskTerminationMasterQueueManager.register("get_task_queue", callable=get_task_queue)
ISCFileTaskTerminationMasterQueueManager.register("get_result_queue", callable=get_result_queue)
manager = ISCFileTaskTerminationMasterQueueManager(address=(config.task_host, config.task_host_port),
authkey=bytes(config.task_host_key, encoding="utf-8"))
manager.start()
task_queue = manager.get_task_queue()
result_queue = manager.get_result_queue()
working_hosts_number = 0
msg_text = "isc task master start, load isc tasks from %s" % (isc_config_file)
logging.info(msg_text)
msg.send_message(msg_text)
isc_tasks_cfg = ISCTaskConfig(isc_config_file, is_use_extended_rules)
isc_tasks = isc_tasks_cfg.isc_tasks
sleep_cnt = 0
online_hosts = set()
for tid in range(len(isc_tasks)):
itask = isc_tasks[tid]
itask.load_isc_task_items()
msg_text = itask.get_isc_task_load_message()
ne_iset = itask.working_ne_iset_numbers
slices = itask.incremental_task_slices[ne_iset]
for s in slices:
task_queue.put((tid, s))
logging.info(msg_text)
msg.send_message(msg_text)
progress_msg_cnt = 10
task_finish = False
while not task_finish:
if sleep_cnt == progress_msg_cnt:
send_itasks_progress_info(isc_tasks, task_queue, working_hosts_number)
sleep_cnt = 0
task_finish = check_itasks_status(isc_tasks, task_queue, working_hosts_number, is_frequent_log)
if result_queue.empty():
time.sleep(sleep_time)
sleep_cnt += 1
continue
whn_diff = process_result_queue(result_queue, isc_tasks)
whn_number = whn_diff[0]
host_ip = whn_diff[1]
working_hosts_number += whn_number
if whn_number == 1:
online_hosts.add(host_ip)
# for itask in isc_tasks:
# transport_files(host_ip, itask.non_se_condition_files)
elif whn_number == -1:
online_hosts.remove(host_ip)
msg_text = "all isc task slices are discatched!"
logging.info(msg_text)
msg.send_message(msg_text)
for i in range(working_hosts_number * 200):
task_queue.put((kill_signal, -1))
while working_hosts_number > 0:
if sleep_cnt == 10:
msg_texts = dump_isc_task_results(isc_tasks)
msg_text = "all isc tasks are discatched, DO NOT add new worker! isc tasks progress info, remain %d task hosts: \n\t\t%s" \
% (working_hosts_number, "\n\t\t".join(msg_texts))
logging.info(msg_text)
msg.send_message(msg_text)
sleep_cnt = 0
if result_queue.empty():
time.sleep(sleep_time)
sleep_cnt += 1
continue
whn_diff = process_result_queue(result_queue, isc_tasks)
working_hosts_number += whn_diff[0]
msg_texts = []
attached_files = []
for it in isc_tasks:
msg_texts.append(it.task_finish())
attached_files.append(it.result_file)
msg_text = "isc tasks finish! \n\t\t%s" % "\n\t\t".join(msg_texts)
logging.info(msg_text)
msg.send_message(msg=msg_text, attached_files=attached_files)
def init_kmn_isc_task_master_from_config(
cls, isc_config_file="isets-tasks.json", sleep_time=30):
start_time = datetime.now()
SearchMasterQueueManger.register("get_task_queue",
callable=cls.get_global_task_queue)
SearchMasterQueueManger.register("get_result_queue",
callable=cls.get_global_result_queue)
manager = SearchMasterQueueManger(address=(config.task_host,
config.task_host_port),
authkey=bytes(config.task_host_key,
encoding="utf-8"))
manager.start()
task_queue = manager.get_task_queue()
result_queue = manager.get_result_queue()
localhost_ip = ssh.get_host_ip()
task_generator = Pool(2)
task_generator.apply_async(cls.itask_slices_generator,
args=(cls, isc_config_file))
task_generator.close()
working_hosts_number = 0
msg_text = "isc task master start, load isc tasks from %s" % (
isc_config_file)
logging.info(msg_text)
msg.send_message(msg_text)
isc_tasks_cfg = ITaskConfig(isc_config_file)
isc_tasks = isc_tasks_cfg.isc_tasks
for itask in isc_tasks:
itask.init_task_numbers()
sleep_cnt = 0
online_hosts = set()
progress_msg_cnt = 10
task_finish = False
print_loop = 100000
print_cnt = 0
while not task_finish:
print_cnt += 1
if print_cnt == print_loop:
cls.send_itasks_progress_info(cls, isc_tasks, task_queue,
working_hosts_number, False)
sleep_cnt = 0
print_cnt = 0
if sleep_cnt == progress_msg_cnt:
cls.send_itasks_progress_info(cls, isc_tasks, task_queue,
working_hosts_number, False)
sleep_cnt = 0
print_cnt = 0
task_finish = cls.check_itasks_status(cls, isc_tasks, online_hosts,
task_queue,
working_hosts_number)
if result_queue.empty():
time.sleep(sleep_time)
sleep_cnt += 1
continue
whn_diff = cls.process_result_queue(cls, result_queue, isc_tasks)
whn_number = whn_diff[0]
host_ip = whn_diff[1]
working_hosts_number += whn_number
if whn_number == 1:
if host_ip != localhost_ip:
online_hosts.add(host_ip)
cls.update_nse_files_to_new_host(host_ip, isc_tasks)
elif whn_number == -1:
if host_ip != localhost_ip:
online_hosts.remove(host_ip)
task_generator.join()
while working_hosts_number > 0:
if sleep_cnt == 10:
cls.send_itasks_progress_info(cls, isc_tasks, task_queue,
working_hosts_number, True)
sleep_cnt = 0
if result_queue.empty():
time.sleep(sleep_time)
sleep_cnt += 1
continue
whn_diff = cls.process_result_queue(cls, result_queue, isc_tasks)
working_hosts_number += whn_diff[0]
msg_texts = []
attached_files = []
for it in isc_tasks:
it.task_finish()
msg_texts.append(it.get_final_detail_progress_info())
attached_files.append(it.result_file)
msg_text = "isc tasks finish! \n\t\t%s" % "\n\t\t".join(msg_texts)
logging.info(msg_text)
msg.send_message(msg=msg_text, attached_files=attached_files)
return isc_tasks
def init_kmn_isc_task_master_from_config(
cls, isc_config_file="isets-tasks.json", sleep_time=30):
manager, task_queue, ht_task_queue, result_queue = \
SearchQueueManager.init_task_master_queue_manager()
manager_tuple = (manager, task_queue, ht_task_queue, result_queue)
localhost_ip = ssh.get_host_ip()
ts_generator_pool = cls.init_task_slices_generator_pool(
cls, isc_config_file)
ht_pool = cls.init_pre_task_worker_pool(cls, isc_config_file,
result_queue)
working_hosts_number = 0
# ht_checking_results = list()
isc_tasks_cfg = ITaskConfig(isc_config_file)
isc_tasks = isc_tasks_cfg.isc_tasks
for itask in isc_tasks:
itask.init_task_numbers()
if os.path.exists(itask.result_file):
os.remove(itask.result_file)
# ht_checking_results.append(list())
msg_text = "isc task master start, load %d isc tasks from %s" % (
len(isc_tasks), isc_config_file)
logging.info(msg_text)
msg.send_message(msg_text)
sleep_cnt = 0
online_hosts = set()
progress_msg_cnt = 10
task_finish = False
print_loop = 100
print_cnt = 0
while not task_finish:
print_cnt += 1
if print_cnt == print_loop:
cls.send_itasks_progress_info(cls, isc_tasks, manager_tuple,
working_hosts_number, False)
sleep_cnt = 0
print_cnt = 0
if sleep_cnt == progress_msg_cnt:
cls.send_itasks_progress_info(cls, isc_tasks, manager_tuple,
working_hosts_number, False)
sleep_cnt = 0
print_cnt = 0
task_finish = cls.check_itasks_status(cls, isc_tasks, online_hosts,
manager_tuple,
working_hosts_number)
if result_queue.empty():
time.sleep(sleep_time)
sleep_cnt += 1
continue
whn_diff = cls.process_result_queue(cls, result_queue, isc_tasks)
whn_number = whn_diff[0]
host_ip = whn_diff[1]
working_hosts_number += whn_number
if whn_number == 1:
if host_ip != localhost_ip:
online_hosts.add(host_ip)
elif whn_number == -1:
if host_ip != localhost_ip:
online_hosts.remove(host_ip)
ts_generator_pool.join()
ht_pool.join()
HTCheckingWorker.send_worker_terminate_info(HTCheckingWorker,
localhost_ip, result_queue)
while working_hosts_number > 0:
if sleep_cnt == 10:
cls.send_itasks_progress_info(cls, isc_tasks, manager_tuple,
working_hosts_number, True)
sleep_cnt = 0
if result_queue.empty():
time.sleep(sleep_time)
sleep_cnt += 1
continue
whn_diff = cls.process_result_queue(cls, result_queue, isc_tasks)
working_hosts_number += whn_diff[0]
msg_text = "isc tasks finish!"
logging.info(msg_text)
msg.send_message(msg=msg_text)
for it in isc_tasks:
it.dump_tmp_se_condition_saving_mem()
msg_text = it.get_final_detail_progress_info()
logging.info(msg_text)
msg.send_message(msg=msg_text)
return isc_tasks
def itask_slices_generator(cls, isc_config_file="isets-tasks.json"):
msg_text = "%s init task slices generator ..." % str(cls)
logging.info(msg_text)
msg.send_message(msg_text)
SearchWorkerQueueManger.register("get_task_queue")
SearchWorkerQueueManger.register("get_result_queue")
manager = SearchWorkerQueueManger(address=(config.task_host,
config.task_host_port),
authkey=bytes(config.task_host_key,
encoding="utf-8"))
manager.connect()
task_queue = manager.get_task_queue()
result_queue = manager.get_result_queue()
isc_tasks_cfg = ITaskConfig(isc_config_file)
isc_tasks = isc_tasks_cfg.isc_tasks
for tid in range(len(isc_tasks)):
it = isc_tasks[tid]
min_ne = it.min_ne
max_ne = it.max_ne
unknown_iset_number = len(it.meta_data.search_space_iset_ids)
rule_number = it.rule_number
left_zone_iset_ids = it.meta_data.search_i4_composed_iset_ids
left_zone_length = len(left_zone_iset_ids)
right_zone_length = unknown_iset_number - left_zone_length
is_use_extended_rules = it.is_use_extended_rules
for i in range(min_ne, max_ne + 1):
ne_iset_number = i
for left_iset_number in range(ne_iset_number + 1):
right_iset_number = ne_iset_number - left_iset_number
if left_iset_number > left_zone_length or right_iset_number > right_zone_length:
continue
task_iter = itertools.combinations(left_zone_iset_ids,
left_iset_number)
for left_ti in task_iter:
left_iset_ids = list(left_ti)
is_contain_semi_valid_rule = iscm.check_contain_rules_without_i_n_iset(
4, left_iset_ids, rule_number,
is_use_extended_rules)
if is_contain_semi_valid_rule:
check_cnt = 0
# C(right_zone_length, right_iset_number)
task_number = CombinaryCounter.compute_comb(
right_zone_length, right_iset_number)
semi_valid_skip_cnt = task_number
stat_item = (ITaskSignal.stat_signal, tid,
ne_iset_number, check_cnt,
task_number, semi_valid_skip_cnt,
None)
result_queue.put(stat_item)
else:
task_item = (tid, (ne_iset_number,
set(left_zone_iset_ids),
left_iset_ids))
# print(task_item)
task_queue.put(task_item)
working_hosts_number = 5
for i in range(working_hosts_number * 200):
task_queue.put((ITaskSignal.kill_signal, -1))
logging.info("all itasks has been dispatched")
def itask_slice_generator_by_i4_meta(ne_iset_number, itask_id, itask,
max_space_size, manager_tuple):
task_queue = manager_tuple[1]
result_queue = manager_tuple[3]
kmn_key = I4RawSearchMaster.get_kmn_meta_key(*itask.k_m_n)
i4_meta = I4RawSearchMaster.i4_meta[kmn_key]
left_zone_length = len(itask.meta_data.search_i4_composed_iset_ids)
search_isets_length = len(itask.meta_data.search_space_iset_ids)
right_zone_length = search_isets_length - left_zone_length
task_slice_cnt = 0
if ne_iset_number <= right_zone_length:
semi_valid_i4_slices_size = CombinaryCounter.compute_comb(
right_zone_length, ne_iset_number)
valid_skip_number = CombinaryCounter.compute_comb(
right_zone_length, ne_iset_number)
result_tuple = (ITaskSignal.stat_signal, itask_id, ne_iset_number,
0, valid_skip_number, valid_skip_number, None)
result_queue.put(result_tuple)
for left_choice in range(1, left_zone_length + 1):
right_choice = ne_iset_number - left_choice
if right_choice > right_zone_length or left_choice > ne_iset_number:
continue
single_slice_right_task_number = CombinaryCounter.compute_comb(
right_zone_length, right_choice)
task_i4_slice_number = max_space_size // single_slice_right_task_number + 1
non_semi_valid_i4_slices_size = i4_meta[left_choice]
itask_sizes = non_semi_valid_i4_slices_size // task_i4_slice_number
if itask_sizes < 92 or itask_sizes > 1200:
itask_sizes = 1200
if itask_sizes > non_semi_valid_i4_slices_size:
itask_sizes = non_semi_valid_i4_slices_size
task_i4_slice_number = non_semi_valid_i4_slices_size // itask_sizes
itask_splitting_points = [
i * task_i4_slice_number for i in range(itask_sizes)
]
if len(itask_splitting_points) == 0:
itask_splitting_points.append(0)
if itask_splitting_points[-1] < non_semi_valid_i4_slices_size:
itask_splitting_points.append(non_semi_valid_i4_slices_size)
for i in range(1, len(itask_splitting_points)):
itask_slice_tuple = (left_choice,
itask_splitting_points[i - 1],
itask_splitting_points[i], right_choice)
itask_slice_tuple = (itask_id, itask_slice_tuple)
task_queue.put(itask_slice_tuple)
task_slice_cnt += 1
total_i4_silces_size = CombinaryCounter.compute_comb(
left_zone_length, left_choice)
semi_valid_i4_slices_size = total_i4_silces_size - non_semi_valid_i4_slices_size
if semi_valid_i4_slices_size > 0:
valid_skip_number = semi_valid_i4_slices_size * single_slice_right_task_number
result_tuple = (ITaskSignal.stat_signal, itask_id,
ne_iset_number, 0, valid_skip_number,
valid_skip_number, None)
result_queue.put(result_tuple)
msg_text = "itask %d-%d-%d ne iset number %d, put %d task slices" % (
*itask.k_m_n, ne_iset_number, task_slice_cnt)
logging.info(msg_text)
msg.send_message(msg_text)