def get_check_itasks_by_non_empty_iset_number_from_autogen(self):
se_iset_ids = self.meta_data.se_iset_ids
unknown_iset_number = len(se_iset_ids)
ne_iset_number = self.working_ne_iset_numbers
max_slice_size = 1000
task_counter = CombinaryCounter(ne_iset_number, unknown_iset_number)
filtered_task_slices = list()
check_icondition_number = 0
task_begin = []
task_length = 0
task_total_number = 0
while True:
task_idx = task_counter.get_current_indicator()
if task_idx is None:
break
task_total_number += 1
if task_length == 0:
task_begin = copy.deepcopy(task_idx)
iset_ids = set()
for t in task_idx:
iset_ids.add(se_iset_ids[t])
if not self.is_contain_non_se_condition(iset_ids):
task_length += 1
if task_length == max_slice_size:
filtered_task_slices.append((task_begin, task_length))
check_icondition_number += task_length
task_length = 0
else:
if task_length > 0:
filtered_task_slices.append((task_begin, task_length))
check_icondition_number += task_length
task_length = 0
if task_length > 0:
filtered_task_slices.append((task_begin, task_length))
check_icondition_number += task_length
self.incremental_task_number[ne_iset_number] = task_total_number
self.task_total_number += task_total_number
nse_icondition_number = task_total_number - check_icondition_number
self.incremental_task_check_number[
ne_iset_number] = check_icondition_number
self.incremental_task_complete_number[
ne_iset_number] = nse_icondition_number
self.incremental_nse_condition_number[
ne_iset_number] = nse_icondition_number
msg_text = "task queue put %d itasks" % len(filtered_task_slices)
return filtered_task_slices, msg_text
def get_check_itasks_by_non_empty_iset_number_from_loaded_isc_slices(self):
ne_iset_number = self.working_ne_iset_numbers
task_slices = self.incremental_task_slices[ne_iset_number]
unknown_iset_number = len(self.meta_data.se_iset_ids)
se_iset_ids = self.meta_data.se_iset_ids
filtered_task_slices = list()
check_icondition_number = 0
for ts in task_slices:
task_counter = CombinaryCounter(ne_iset_number,
unknown_iset_number)
task_counter.reset_current_indicator(ts[0])
task_begin = copy.deepcopy(ts[0])
task_length = 0
for i in range(ts[1]):
task_idx = task_counter.get_current_indicator()
if task_length == 0:
task_begin = copy.deepcopy(task_idx)
iset_ids = set()
for t in task_idx:
iset_ids.add(se_iset_ids[t])
if not self.is_contain_non_se_condition(iset_ids):
task_length += 1
else:
if task_length > 0:
filtered_task_slices.append((task_begin, task_length))
check_icondition_number += task_length
task_length = 0
# task_idx = task_counter.get_current_indicator()
# task_begin = copy.deepcopy(task_idx)
if task_length > 0:
filtered_task_slices.append((task_begin, task_length))
check_icondition_number += task_length
nse_icondition_number = self.incremental_task_number[
ne_iset_number] - check_icondition_number
self.incremental_task_check_number[
ne_iset_number] = check_icondition_number
self.incremental_task_complete_number[
ne_iset_number] = nse_icondition_number
self.incremental_nse_condition_number[
ne_iset_number] = nse_icondition_number
msg_text = "task queue put %d itasks" % len(filtered_task_slices)
return filtered_task_slices, msg_text
def itask_slices_generator(isc_config_file="isets-tasks.json",
is_use_extended_rules=True):
ISCFileTaskTerminationWorkerQueueManager.register("get_task_queue")
ISCFileTaskTerminationWorkerQueueManager.register("get_result_queue")
manager = ISCFileTaskTerminationWorkerQueueManager(
address=(config.task_host, config.task_host_port),
authkey=bytes(config.task_host_key, encoding="utf-8"))
manager.connect()
task_queue = manager.get_task_queue()
isc_tasks_cfg = ISCTaskConfig(isc_config_file, is_use_extended_rules)
isc_tasks = isc_tasks_cfg.isc_tasks
for tid in range(len(isc_tasks)):
it = isc_tasks[tid]
min_ne = it.min_ne
max_ne = it.max_ne
se_iset_ids = it.meta_data.se_iset_ids
unknown_iset_number = len(se_iset_ids)
max_task_slice_number = 10000
for i in range(min_ne, max_ne + 1):
task_counter = CombinaryCounter(i, unknown_iset_number)
task_start_idx = []
task_idx_cnt = 0
while True:
task_end_idx = task_counter.get_current_indicator()
if task_end_idx is None:
break
if task_idx_cnt == 0:
task_start_idx = copy.deepcopy(task_end_idx)
task_idx_cnt += 1
if task_idx_cnt == max_task_slice_number:
task_queue.put((tid, (task_start_idx, task_idx_cnt)))
task_idx_cnt = 0
if task_idx_cnt != 0:
task_queue.put((tid, (task_start_idx, task_idx_cnt)))
working_hosts_number = 5
for i in range(working_hosts_number * 200):
task_queue.put((kill_signal, -1))
logging.info("all itasks has been dispatched")
def generate_isp_slices_task_queue(max_task_slice_size,
min_non_empty_iset_number,
max_non_empty_iset_number,
unknown_iset_number):
task_total_number = 0
task_slice_number = 0
task_queue = list()
for ne_number in range(min_non_empty_iset_number,
max_non_empty_iset_number + 1):
task_counter = CombinaryCounter(ne_number, unknown_iset_number)
task_start_idx = []
task_idx_cnt = 0
while True:
task_end_idx = task_counter.get_current_indicator()
if task_end_idx is None:
break
if task_idx_cnt == 0:
task_start_idx = copy.deepcopy(task_end_idx)
task_idx_cnt += 1
if task_idx_cnt == max_task_slice_size:
task_queue.append((task_start_idx, task_idx_cnt))
task_total_number += task_idx_cnt
task_slice_number += 1
task_idx_cnt = 0
if task_idx_cnt != 0:
task_queue.append((task_start_idx, task_idx_cnt))
task_total_number += task_idx_cnt
task_slice_number += 1
return task_queue, task_total_number, task_slice_number
def kmn_isc_task_worker(isc_config_file="isets-tasks.json", worker_name="", lp_type="lpmln", is_check_valid_rules=True, is_use_extended_rules=True):
ISCFileTaskTerminationWorkerQueueManager.register("get_task_queue")
ISCFileTaskTerminationWorkerQueueManager.register("get_result_queue")
manager = ISCFileTaskTerminationWorkerQueueManager(address=(config.task_host, config.task_host_port),
authkey=bytes(config.task_host_key, encoding="utf-8"))
manager.connect()
task_queue = manager.get_task_queue()
result_queue = manager.get_result_queue()
time_fmt = "%Y-%m-%d %H:%M:%S.%f"
worker_host_name = config.worker_host_name
msg_text = "task worker %s start!" % (worker_name)
logging.info(msg_text)
isc_tasks = ISCTaskConfig(isc_config_file, is_use_extended_rules)
isc_tasks = isc_tasks.isc_tasks
processed_task_slices_number = 0
while True:
if not pathlib.Path(config.task_host_lock_file).exists():
break
if task_queue.empty():
time.sleep(20)
continue
itask = task_queue.get()
if itask[0] == kill_signal:
msg_text = "%s:%s isc task worker terminate ..." % (worker_host_name, worker_name)
logging.info(msg_text)
break
start_time = datetime.now()
start_time_str = start_time.strftime(time_fmt)[:-3]
isc_task_id = itask[0]
it = isc_tasks[isc_task_id]
k_size = it.k_m_n[0]
m_size = it.k_m_n[1]
n_size = it.k_m_n[2]
se_iset_ids = it.meta_data.se_iset_ids
unknown_iset_number = len(se_iset_ids)
task_details = itask[1]
isc_begin = copy.deepcopy(task_details[0])
task_start = task_details[0]
task_start = [str(s) for s in task_start]
task_start = ",".join(task_start)
task_number = task_details[1]
task_name = worker_name + ("-task-%d" % processed_task_slices_number)
ne_number = len(isc_begin)
msg_text = "%s: %d-%d-%d isc task: from %s length %d, nonempty iset number %d" % (
task_name, k_size, m_size, n_size, task_start, task_number, ne_number)
logging.info(msg_text)
task_counter = CombinaryCounter(ne_number, unknown_iset_number)
task_counter.reset_current_indicator(isc_begin)
se_cdt_cnt = 0
nse_cdt_cnt = 0
se_conditions_cache = list()
nse_conditions_cache = list()
validator = ISetConditionValidator(lp_type=lp_type, is_use_extended_rules=is_use_extended_rules)
for i in range(task_number):
task_idx = task_counter.get_current_indicator()
non_ne_ids = set()
for t in task_idx:
non_ne_ids.add(se_iset_ids[t])
is_contain_valid_rule, is_strongly_equivalent, condition = \
validator.validate_kmn_extended_iset_condition_from_non_emtpy_iset_ids_return_icondition_obj(
non_ne_ids, k_size, m_size, n_size, is_check_valid_rule=is_check_valid_rules)
if not is_contain_valid_rule:
if is_strongly_equivalent:
se_conditions_cache.append(condition)
se_cdt_cnt += 1
else:
nse_conditions_cache.append(condition)
nse_cdt_cnt += 1
# for sec in se_conditions_cache:
result_queue.put((nse_condition_signal, isc_task_id, nse_conditions_cache))
result_queue.put((se_condition_signal, isc_task_id, se_conditions_cache))
end_time = datetime.now()
end_time_str = end_time.strftime(time_fmt)[:-3]
msg_text = "%s, end %d-%d-%d isc tasks from %s length %d, start time %s, end time %s, find %d se conditions (no semi-valid rules), find %d non-se conditions" % (
task_name, k_size, m_size, n_size, task_start, task_number, start_time_str, end_time_str, se_cdt_cnt, nse_cdt_cnt)
logging.info(msg_text)
result_queue.put((stat_signal, isc_task_id, ne_number, task_number, (start_time, end_time)))
processed_task_slices_number += 1
logging.info("%s processes %d isc task slices" % (worker_name, processed_task_slices_number))