def itask_slices_generator(isc_config_file="isets-tasks.json",
is_use_extended_rules=True):
ISCFileTaskTerminationWorkerQueueManager.register("get_task_queue")
ISCFileTaskTerminationWorkerQueueManager.register("get_result_queue")
manager = ISCFileTaskTerminationWorkerQueueManager(
address=(config.task_host, config.task_host_port),
authkey=bytes(config.task_host_key, encoding="utf-8"))
manager.connect()
task_queue = manager.get_task_queue()
isc_tasks_cfg = ISCTaskConfig(isc_config_file, is_use_extended_rules)
isc_tasks = isc_tasks_cfg.isc_tasks
for tid in range(len(isc_tasks)):
it = isc_tasks[tid]
min_ne = it.min_ne
max_ne = it.max_ne
se_iset_ids = it.meta_data.se_iset_ids
unknown_iset_number = len(se_iset_ids)
max_task_slice_number = 10000
for i in range(min_ne, max_ne + 1):
task_counter = CombinaryCounter(i, unknown_iset_number)
task_start_idx = []
task_idx_cnt = 0
while True:
task_end_idx = task_counter.get_current_indicator()
if task_end_idx is None:
break
if task_idx_cnt == 0:
task_start_idx = copy.deepcopy(task_end_idx)
task_idx_cnt += 1
if task_idx_cnt == max_task_slice_number:
task_queue.put((tid, (task_start_idx, task_idx_cnt)))
task_idx_cnt = 0
if task_idx_cnt != 0:
task_queue.put((tid, (task_start_idx, task_idx_cnt)))
working_hosts_number = 5
for i in range(working_hosts_number * 200):
task_queue.put((kill_signal, -1))
logging.info("all itasks has been dispatched")
def itask_slices_generator(isc_config_file="isets-tasks.json",
is_use_extended_rules=True):
ISCFileTaskTerminationWorkerQueueManager.register("get_task_queue")
ISCFileTaskTerminationWorkerQueueManager.register("get_result_queue")
manager = ISCFileTaskTerminationWorkerQueueManager(
address=(config.task_host, config.task_host_port),
authkey=bytes(config.task_host_key, encoding="utf-8"))
manager.connect()
task_queue = manager.get_task_queue()
isc_tasks_cfg = ISCTaskConfig(isc_config_file, is_use_extended_rules)
isc_tasks = isc_tasks_cfg.isc_tasks
for tid in range(len(isc_tasks)):
it = isc_tasks[tid]
min_ne = it.min_ne
max_ne = it.max_ne
se_iset_ids = it.meta_data.se_iset_ids
unknown_iset_number = len(se_iset_ids)
for i in range(min_ne, max_ne + 1):
ne_iset_number = i
left_length = int(unknown_iset_number / 2)
if left_length > 12:
left_length = 12
right_length = unknown_iset_number - left_length
left_iset_ids = se_iset_ids[0:left_length]
for left_iset_number in range(ne_iset_number + 1):
right_iset_number = ne_iset_number - left_iset_number
if left_iset_number > left_length or right_iset_number > right_length:
continue
task_iter = itertools.combinations(left_iset_ids,
left_iset_number)
for left_ti in task_iter:
task_item = (tid, (ne_iset_number, left_length,
list(left_ti)))
task_queue.put(task_item)
working_hosts_number = 5
for i in range(working_hosts_number * 200):
task_queue.put((kill_signal, -1))
logging.info("all itasks has been dispatched")
def init_kmn_isc_task_workers(isc_config_file="isets-tasks.json",
lp_type="lpmln",
is_check_valid_rules=True,
is_use_extended_rules=True):
payload = config.worker_payload
worker_pool = Pool(payload)
pathlib.Path(config.task_host_lock_file).touch()
ISCFileTaskTerminationWorkerQueueManager.register("get_task_queue")
ISCFileTaskTerminationWorkerQueueManager.register("get_result_queue")
manager = ISCFileTaskTerminationWorkerQueueManager(
address=(config.task_host, config.task_host_port),
authkey=bytes(config.task_host_key, encoding="utf-8"))
manager.connect()
result_queue = manager.get_result_queue()
host_ip = ssh.get_host_ip()
result_queue.put((add_worker_signal, config.worker_host_name, host_ip))
logging.info("task worker host %s start ..." % config.worker_host_name)
# 初始化不等价条件目录文件
isc_tasks = ISCTaskConfig(isc_config_file, is_use_extended_rules)
isc_tasks = isc_tasks.isc_tasks
init_worker_host_nse_envs(isc_tasks)
for i in range(payload):
worker_pool.apply_async(kmn_isc_task_worker,
args=(isc_config_file, i + 1, lp_type,
is_check_valid_rules,
is_use_extended_rules))
worker_pool.close()
worker_pool.join()
# if pathlib.Path(task_worker_host_lock_file).exists():
result_queue.put((kill_signal, config.worker_host_name, host_ip))
logging.info("task worker host %s send kill signal ..." %
config.worker_host_name)
logging.info("task worker host %s exit ..." % config.worker_host_name)
def kmn_isc_task_worker(isc_config_file="isets-tasks.json", worker_name="", lp_type="lpmln", is_check_valid_rules=True, is_use_extended_rules=True):
ISCFileTaskTerminationWorkerQueueManager.register("get_task_queue")
ISCFileTaskTerminationWorkerQueueManager.register("get_result_queue")
manager = ISCFileTaskTerminationWorkerQueueManager(address=(config.task_host, config.task_host_port),
authkey=bytes(config.task_host_key, encoding="utf-8"))
manager.connect()
task_queue = manager.get_task_queue()
result_queue = manager.get_result_queue()
time_fmt = "%Y-%m-%d %H:%M:%S.%f"
worker_host_name = config.worker_host_name
msg_text = "task worker %s start!" % (worker_name)
logging.info(msg_text)
isc_tasks = ISCTaskConfig(isc_config_file, is_use_extended_rules)
isc_tasks = isc_tasks.isc_tasks
processed_task_slices_number = 0
while True:
if not pathlib.Path(config.task_host_lock_file).exists():
break
if task_queue.empty():
time.sleep(20)
continue
itask = task_queue.get()
if itask[0] == kill_signal:
msg_text = "%s:%s isc task worker terminate ..." % (worker_host_name, worker_name)
logging.info(msg_text)
break
start_time = datetime.now()
start_time_str = start_time.strftime(time_fmt)[:-3]
isc_task_id = itask[0]
it = isc_tasks[isc_task_id]
k_size = it.k_m_n[0]
m_size = it.k_m_n[1]
n_size = it.k_m_n[2]
se_iset_ids = it.meta_data.se_iset_ids
unknown_iset_number = len(se_iset_ids)
task_details = itask[1]
isc_begin = copy.deepcopy(task_details[0])
task_start = task_details[0]
task_start = [str(s) for s in task_start]
task_start = ",".join(task_start)
task_number = task_details[1]
task_name = worker_name + ("-task-%d" % processed_task_slices_number)
ne_number = len(isc_begin)
msg_text = "%s: %d-%d-%d isc task: from %s length %d, nonempty iset number %d" % (
task_name, k_size, m_size, n_size, task_start, task_number, ne_number)
logging.info(msg_text)
task_counter = CombinaryCounter(ne_number, unknown_iset_number)
task_counter.reset_current_indicator(isc_begin)
se_cdt_cnt = 0
nse_cdt_cnt = 0
se_conditions_cache = list()
nse_conditions_cache = list()
validator = ISetConditionValidator(lp_type=lp_type, is_use_extended_rules=is_use_extended_rules)
for i in range(task_number):
task_idx = task_counter.get_current_indicator()
non_ne_ids = set()
for t in task_idx:
non_ne_ids.add(se_iset_ids[t])
is_contain_valid_rule, is_strongly_equivalent, condition = \
validator.validate_kmn_extended_iset_condition_from_non_emtpy_iset_ids_return_icondition_obj(
non_ne_ids, k_size, m_size, n_size, is_check_valid_rule=is_check_valid_rules)
if not is_contain_valid_rule:
if is_strongly_equivalent:
se_conditions_cache.append(condition)
se_cdt_cnt += 1
else:
nse_conditions_cache.append(condition)
nse_cdt_cnt += 1
# for sec in se_conditions_cache:
result_queue.put((nse_condition_signal, isc_task_id, nse_conditions_cache))
result_queue.put((se_condition_signal, isc_task_id, se_conditions_cache))
end_time = datetime.now()
end_time_str = end_time.strftime(time_fmt)[:-3]
msg_text = "%s, end %d-%d-%d isc tasks from %s length %d, start time %s, end time %s, find %d se conditions (no semi-valid rules), find %d non-se conditions" % (
task_name, k_size, m_size, n_size, task_start, task_number, start_time_str, end_time_str, se_cdt_cnt, nse_cdt_cnt)
logging.info(msg_text)
result_queue.put((stat_signal, isc_task_id, ne_number, task_number, (start_time, end_time)))
processed_task_slices_number += 1
logging.info("%s processes %d isc task slices" % (worker_name, processed_task_slices_number))
def init_kmn_isc_task_master_from_config(isc_config_file="isets-tasks.json", sleep_time=30, is_use_extended_rules=True, is_frequent_log=False):
start_time = datetime.now()
ISCFileTaskTerminationMasterQueueManager.register("get_task_queue", callable=get_task_queue)
ISCFileTaskTerminationMasterQueueManager.register("get_result_queue", callable=get_result_queue)
manager = ISCFileTaskTerminationMasterQueueManager(address=(config.task_host, config.task_host_port),
authkey=bytes(config.task_host_key, encoding="utf-8"))
manager.start()
task_queue = manager.get_task_queue()
result_queue = manager.get_result_queue()
working_hosts_number = 0
msg_text = "isc task master start, load isc tasks from %s" % (isc_config_file)
logging.info(msg_text)
msg.send_message(msg_text)
isc_tasks_cfg = ISCTaskConfig(isc_config_file, is_use_extended_rules)
isc_tasks = isc_tasks_cfg.isc_tasks
sleep_cnt = 0
online_hosts = set()
for tid in range(len(isc_tasks)):
itask = isc_tasks[tid]
itask.load_isc_task_items()
msg_text = itask.get_isc_task_load_message()
ne_iset = itask.working_ne_iset_numbers
slices = itask.incremental_task_slices[ne_iset]
for s in slices:
task_queue.put((tid, s))
logging.info(msg_text)
msg.send_message(msg_text)
progress_msg_cnt = 10
task_finish = False
while not task_finish:
if sleep_cnt == progress_msg_cnt:
send_itasks_progress_info(isc_tasks, task_queue, working_hosts_number)
sleep_cnt = 0
task_finish = check_itasks_status(isc_tasks, task_queue, working_hosts_number, is_frequent_log)
if result_queue.empty():
time.sleep(sleep_time)
sleep_cnt += 1
continue
whn_diff = process_result_queue(result_queue, isc_tasks)
whn_number = whn_diff[0]
host_ip = whn_diff[1]
working_hosts_number += whn_number
if whn_number == 1:
online_hosts.add(host_ip)
# for itask in isc_tasks:
# transport_files(host_ip, itask.non_se_condition_files)
elif whn_number == -1:
online_hosts.remove(host_ip)
msg_text = "all isc task slices are discatched!"
logging.info(msg_text)
msg.send_message(msg_text)
for i in range(working_hosts_number * 200):
task_queue.put((kill_signal, -1))
while working_hosts_number > 0:
if sleep_cnt == 10:
msg_texts = dump_isc_task_results(isc_tasks)
msg_text = "all isc tasks are discatched, DO NOT add new worker! isc tasks progress info, remain %d task hosts: \n\t\t%s" \
% (working_hosts_number, "\n\t\t".join(msg_texts))
logging.info(msg_text)
msg.send_message(msg_text)
sleep_cnt = 0
if result_queue.empty():
time.sleep(sleep_time)
sleep_cnt += 1
continue
whn_diff = process_result_queue(result_queue, isc_tasks)
working_hosts_number += whn_diff[0]
msg_texts = []
attached_files = []
for it in isc_tasks:
msg_texts.append(it.task_finish())
attached_files.append(it.result_file)
msg_text = "isc tasks finish! \n\t\t%s" % "\n\t\t".join(msg_texts)
logging.info(msg_text)
msg.send_message(msg=msg_text, attached_files=attached_files)
def init_kmn_isc_task_master_from_config(isc_config_file="isets-tasks.json",
sleep_time=30,
is_use_extended_rules=True,
is_frequent_log=False):
start_time = datetime.now()
ISCFileTaskTerminationMasterQueueManager.register("get_task_queue",
callable=get_task_queue)
ISCFileTaskTerminationMasterQueueManager.register(
"get_result_queue", callable=get_result_queue)
manager = ISCFileTaskTerminationMasterQueueManager(
address=(config.task_host, config.task_host_port),
authkey=bytes(config.task_host_key, encoding="utf-8"))
manager.start()
task_queue = manager.get_task_queue()
result_queue = manager.get_result_queue()
localhost_ip = ssh.get_host_ip()
task_generator = Pool(2)
task_generator.apply_async(itask_slices_generator,
args=(isc_config_file, is_use_extended_rules))
task_generator.close()
working_hosts_number = 0
msg_text = "isc task master start, load isc tasks from %s" % (
isc_config_file)
logging.info(msg_text)
msg.send_message(msg_text)
isc_tasks_cfg = ISCTaskConfig(isc_config_file, is_use_extended_rules)
isc_tasks = isc_tasks_cfg.isc_tasks
for itask in isc_tasks:
itask.init_task_numbers()
sleep_cnt = 0
online_hosts = set()
progress_msg_cnt = 10
task_finish = False
print_loop = 100000
print_cnt = 0
while not task_finish:
print_cnt += 1
if print_cnt == print_loop:
send_itasks_progress_info(isc_tasks, task_queue,
working_hosts_number)
sleep_cnt = 0
print_cnt = 0
if sleep_cnt == progress_msg_cnt:
send_itasks_progress_info(isc_tasks, task_queue,
working_hosts_number)
sleep_cnt = 0
print_cnt = 0
task_finish = check_itasks_status(isc_tasks, online_hosts, task_queue,
working_hosts_number)
if result_queue.empty():
time.sleep(sleep_time)
sleep_cnt += 1
continue
whn_diff = process_result_queue(result_queue, isc_tasks)
whn_number = whn_diff[0]
host_ip = whn_diff[1]
working_hosts_number += whn_number
if whn_number == 1:
if host_ip != localhost_ip:
online_hosts.add(host_ip)
update_nse_files_to_new_host(host_ip, isc_tasks)
elif whn_number == -1:
if host_ip != localhost_ip:
online_hosts.remove(host_ip)
msg_text = "all isc task slices are discatched!"
logging.info(msg_text)
msg.send_message(msg_text)
task_generator.join()
while working_hosts_number > 0:
if sleep_cnt == 10:
msg_texts = dump_isc_task_results(isc_tasks)
msg_text = "all isc tasks are discatched, DO NOT add new worker! isc tasks progress info, remain %d task hosts: \n\t\t%s" \
% (working_hosts_number, "\n\t\t".join(msg_texts))
logging.info(msg_text)
msg.send_message(msg_text)
sleep_cnt = 0
if result_queue.empty():
time.sleep(sleep_time)
sleep_cnt += 1
continue
whn_diff = process_result_queue(result_queue, isc_tasks)
working_hosts_number += whn_diff[0]
msg_texts = []
attached_files = []
for it in isc_tasks:
it.task_finish()
msg_texts.append(it.get_final_detail_progress_info())
attached_files.append(it.result_file)
msg_text = "isc tasks finish! \n\t\t%s" % "\n\t\t".join(msg_texts)
logging.info(msg_text)
msg.send_message(msg=msg_text, attached_files=attached_files)
return isc_tasks
def kmn_isc_task_worker(isc_config_file="isets-tasks.json",
worker_id=1,
lp_type="lpmln",
is_check_valid_rules=True,
is_use_extended_rules=True):
ISCFileTaskTerminationWorkerQueueManager.register("get_task_queue")
ISCFileTaskTerminationWorkerQueueManager.register("get_result_queue")
manager = ISCFileTaskTerminationWorkerQueueManager(
address=(config.task_host, config.task_host_port),
authkey=bytes(config.task_host_key, encoding="utf-8"))
is_check_valid_rules = False
manager.connect()
task_queue = manager.get_task_queue()
result_queue = manager.get_result_queue()
worker_name = "worker-%d" % worker_id
time_fmt = "%Y-%m-%d %H:%M:%S.%f"
worker_host_name = config.worker_host_name
msg_text = "task worker %s start!" % (worker_name)
logging.info(msg_text)
isc_tasks = ISCTaskConfig(isc_config_file, is_use_extended_rules)
isc_tasks = isc_tasks.isc_tasks
processed_task_slices_number = 0
for it in isc_tasks:
it.loaded_non_se_condition_files.add(1)
first_print_debug_log = True
while True:
if not pathlib.Path(config.task_host_lock_file).exists():
break
if task_queue.empty():
if first_print_debug_log:
logging.info("waiting for isc task slices")
first_print_debug_log = False
time.sleep(20)
continue
itask = task_queue.get()
if itask[0] == kill_signal:
msg_text = "%s:%s isc task worker terminate ..." % (
worker_host_name, worker_name)
logging.info(msg_text)
break
isc_task_id = itask[0]
task_params = itask[1]
ne_iset_number = task_params[0]
left_length = task_params[1]
left_iset_ids = task_params[2]
it = isc_tasks[isc_task_id]
task_terminate_flag = isnse.get_task_early_terminate_flag_file(
*it.k_m_n)
nse_iset_number = ne_iset_number - 1
if nse_iset_number not in it.loaded_non_se_condition_files:
load_complete = False
while not load_complete:
if pathlib.Path(task_terminate_flag).exists():
it.is_task_finish = True
break
load_complete = task_worker_load_nse_conditions(
it, nse_iset_number)
if it.is_task_finish:
continue
start_time = datetime.now()
start_time_str = start_time.strftime(time_fmt)[:-3]
k_size = it.k_m_n[0]
m_size = it.k_m_n[1]
n_size = it.k_m_n[2]
se_iset_ids = it.meta_data.se_iset_ids
right_iset_ids = se_iset_ids[left_length:]
right_iset_number = ne_iset_number - len(left_iset_ids)
# unknown_iset_number = len(se_iset_ids)
task_name = worker_name + ("-task-%d" % processed_task_slices_number)
msg_text = "%s: %d-%d-%d isc task: nonempty iset number %d, left zone length %d, left isets {%s}" % (
task_name, k_size, m_size, n_size, ne_iset_number, left_length,
join_list_data(left_iset_ids))
logging.info(msg_text)
se_cdt_cnt = 0
nse_cdt_cnt = 0
new_nse_cdt_cnt = 0
check_cdt_cnt = 0
se_conditions_cache = list()
nse_conditions_cache = list()
validator = ISetConditionValidator(
lp_type=lp_type, is_use_extended_rules=is_use_extended_rules)
task_iter = itertools.combinations(right_iset_ids, right_iset_number)
task_number = 0
for right_ti in task_iter:
non_ne_ids = list()
non_ne_ids.extend(left_iset_ids)
non_ne_ids.extend(list(right_ti))
non_ne_ids = set(non_ne_ids)
task_number += 1
if check_contain_nse_subparts(non_ne_ids, it):
nse_cdt_cnt += 1
continue
check_cdt_cnt += 1
if not check_contain_i4_isets(non_ne_ids, it):
continue
is_contain_valid_rule, is_strongly_equivalent, condition = \
validator.validate_kmn_extended_iset_condition_from_non_emtpy_iset_ids_return_icondition_obj(
non_ne_ids, k_size, m_size, n_size, is_check_valid_rule=is_check_valid_rules)
# if not is_contain_valid_rule:
if is_strongly_equivalent:
se_conditions_cache.append(condition)
se_cdt_cnt += 1
else:
nse_conditions_cache.append(condition)
nse_cdt_cnt += 1
new_nse_cdt_cnt += 1
# for sec in se_conditions_cache:
if se_cdt_cnt > 0:
result_queue.put(
(se_condition_signal, isc_task_id, se_conditions_cache))
if new_nse_cdt_cnt > 0:
result_queue.put(
(nse_condition_signal, isc_task_id, nse_conditions_cache))
end_time = datetime.now()
end_time_str = end_time.strftime(time_fmt)[:-3]
msg_text = "%s: %d-%d-%d isc task: nonempty iset number %d, left zone length %d, left isets {%s}, start time %s, end time %s, find %d se conditions (no semi-valid rules), find %d non-se conditions" % (
task_name, k_size, m_size, n_size, ne_iset_number, left_length,
join_list_data(left_iset_ids), start_time_str, end_time_str,
se_cdt_cnt, nse_cdt_cnt)
logging.info(msg_text)
result_queue.put((stat_signal, isc_task_id, ne_iset_number,
check_cdt_cnt, task_number, (start_time, end_time)))
processed_task_slices_number += 1
first_print_debug_log = True
logging.info("%s processes %d isc task slices" %
(worker_name, processed_task_slices_number))