def start_reduce(): files = os.listdir(map_directory) file_groups = array_split(files, nodes_reduce) threads = [None] * len(file_groups) states = [True] * len(file_groups) for i, files in enumerate(file_groups): threads[i] = Thread(target=reduce, args=[files, i, states]) threads[i].args = [files, i, states] threads[i].start() while True: for thread in threads: thread.join() if not_dead(states): break for i, (thread, state) in enumerate(zip(threads, states)): if not state: id_new, t = find_alive(threads, states) transfer_task(new_thread=t, task=thread.args, old=i, new=id_new, func=reduce) if all_dead(states): raise Exception('All reduce nodes have failed!') print('Reduce finished.\n')
def start_map(): files = os.listdir(text_directory) file_groups = array_split(files, nodes_map) threads = [None] * len(file_groups) states = [True] * len(file_groups) locks = {} while True: for i, files in enumerate(file_groups): threads[i] = Thread(target=map, args=[files, i, states, locks]) threads[i].args = [files, i, states] threads[i].start() for thread in threads: thread.join() if not_dead(states): break clear_map() print('Restarting mapping tasks.\n') print('Mapping finished.\n')
def update_thread_loop(thread: threading.Thread, loop: asyncio.AbstractEventLoop) -> None: thread.args = (loop, )