def start_reduce():
files = os.listdir(map_directory)
file_groups = array_split(files, nodes_reduce)
threads = [None] * len(file_groups)
states = [True] * len(file_groups)
for i, files in enumerate(file_groups):
threads[i] = Thread(target=reduce, args=[files, i, states])
threads[i].args = [files, i, states]
threads[i].start()
while True:
for thread in threads:
thread.join()
if not_dead(states):
break
for i, (thread, state) in enumerate(zip(threads, states)):
if not state:
id_new, t = find_alive(threads, states)
transfer_task(new_thread=t,
task=thread.args,
old=i,
new=id_new,
func=reduce)
if all_dead(states):
raise Exception('All reduce nodes have failed!')
print('Reduce finished.\n')
def start_map():
files = os.listdir(text_directory)
file_groups = array_split(files, nodes_map)
threads = [None] * len(file_groups)
states = [True] * len(file_groups)
locks = {}
while True:
for i, files in enumerate(file_groups):
threads[i] = Thread(target=map, args=[files, i, states, locks])
threads[i].args = [files, i, states]
threads[i].start()
for thread in threads:
thread.join()
if not_dead(states):
break
clear_map()
print('Restarting mapping tasks.\n')
print('Mapping finished.\n')
def update_thread_loop(thread: threading.Thread,
loop: asyncio.AbstractEventLoop) -> None:
thread.args = (loop, )
