def itask_slices_generator(cls, isc_config_file):
max_space_size = 10000000000
msg_text = "%s init task slices generator ..." % str(cls)
logging.info(msg_text)
msg.send_message(msg_text)
manager_tuple = SearchQueueManager.init_task_worker_queue_manager()
task_queue = manager_tuple[1]
isc_tasks_cfg = ITaskConfig(isc_config_file)
isc_tasks = isc_tasks_cfg.isc_tasks
for tid in range(len(isc_tasks)):
it = isc_tasks[tid]
min_ne = it.min_ne
max_ne = it.max_ne
rule_number = sum(it.k_m_n)
isnse.clear_task_space_layer_finish_flag_files(
*it.k_m_n, min_ne, max_ne)
for ne_iset_number in range(min_ne, max_ne + 1):
msg_text = "generating %d-%d-%d %d layer task slices" % (
*it.k_m_n, ne_iset_number)
logging.info(msg_text)
cls.itask_slice_generator_by_i4_meta(ne_iset_number, tid, it,
max_space_size,
manager_tuple)
# if ne_iset_number <= rule_number:
# cls.itask_slice_generator_by_i4_meta(ne_iset_number, tid, it, max_space_size, manager_tuple)
# else:
# if not cls.check_itask_terminate_status(it):
# flag_file = isnse.get_task_space_layer_finish_flag_file(*it.k_m_n, ne_iset_number - 2)
# while not pathlib.Path(flag_file).exists():
# if cls.check_itask_terminate_status(it):
# break
# time.sleep(1)
#
# cls.itask_slice_generator_by_i4_meta(ne_iset_number, tid, it, max_space_size, manager_tuple)
working_hosts_number = 5
for i in range(working_hosts_number * 200):
task_queue.put((ITaskSignal.kill_signal, -1))
logging.info("all itasks has been dispatched")
def itask_slices_generator(cls, isc_config_file):
max_space_size = 100000000000
msg_text = "%s init task slices generator ..." % str(cls)
logging.info(msg_text)
msg.send_message(msg_text)
manager, task_queue, ht_task_queue, result_queue = \
SearchQueueManager.init_task_worker_queue_manager()
isc_tasks_cfg = ITaskConfig(isc_config_file)
isc_tasks = isc_tasks_cfg.isc_tasks
for tid in range(len(isc_tasks)):
it = isc_tasks[tid]
min_ne = it.min_ne
max_ne = it.max_ne
isnse.clear_task_space_layer_finish_flag_files(*it.k_m_n, min_ne, max_ne)
left_zone_length = len(it.meta_data.search_i4_composed_iset_ids)
search_isets = copy.deepcopy(it.meta_data.search_space_iset_ids)
search_isets_length = len(search_isets)
max_left_zone_length = 12
if left_zone_length > max_left_zone_length:
left_zone_length = 12
rule_number = sum(it.k_m_n)
left_zone_iset_ids = search_isets[0:left_zone_length]
right_zone_iset_ids = search_isets[left_zone_length:]
for ne_iset_number in range(min_ne, max_ne + 1):
msg_text = "generating %d-%d-%d %d layer task slices" % (*it.k_m_n, ne_iset_number)
logging.info(msg_text)
if ne_iset_number <= rule_number:
left_split = True
task_slices = CombinationSearchingSpaceSplitter.vandermonde_generator(
left_zone_iset_ids, right_zone_iset_ids, ne_iset_number)
for ts in task_slices:
new_ts = (left_split, set(ts[0]), left_zone_length, ts[2])
task_queue.put((tid, new_ts))
else:
if not cls.check_itask_terminate_status(it):
flag_file = isnse.get_task_space_layer_finish_flag_file(*it.k_m_n, ne_iset_number - 2)
while not pathlib.Path(flag_file).exists():
if cls.check_itask_terminate_status(it):
break
time.sleep(1)
task_slices = CombinationSearchingSpaceSplitter.merge_small_near_uniform_vandermonde_generator(
left_zone_iset_ids, right_zone_iset_ids, ne_iset_number, max_space_size=max_space_size)
ts_cnt = 0
for ts in task_slices:
task_queue.put((tid, ts))
ts_cnt += 1
if ts_cnt % 10000 == 0 and cls.check_itask_terminate_status(it):
break
working_hosts_number = 5
for i in range(working_hosts_number * 200):
task_queue.put((ITaskSignal.kill_signal, -1))
logging.info("all itasks has been dispatched")