def main(): file_content = file_reader('./books/') print('Files read') client = Client('tcp://192.168.1.40:8786') dsk = { 'reader': file_content, 'mapper': (mapper, 'reader'), 'sorter': (sorter, 'mapper'), 'reducer': (reducer, 'sorter'), 'presenter': (presenter, 'reducer') } result = client.get(dsk, 'presenter') for word in result: print(word)
def main(): file_content = open('177.txt', encoding='ISO-8859-1').readlines() # result = mapper(file_content) # result = sorter(result) # result = reducer(result) client = Client('tcp://192.168.1.40:8786') dsk = { 'content': file_content, 'mapper': (mapper, 'content'), 'sorter': (sorter, 'mapper'), 'reducer': (reducer, 'sorter') } result = client.get(dsk, 'reducer') for line in result: print(line)
def main(): file_content, book_count = file_reader('./books/') start = time.time() client = Client('tcp://192.168.1.64:8786') dsk = { 'content': file_content, 'mapper': (mapper, 'content'), 'sorter': (sorter, 'mapper'), 'reducer': (reducer, 'sorter'), 'presenter': (presenter, 'reducer') } result = client.get(dsk, 'presenter') end = time.time() for line in result: print(line) print( f'Counted the words in {book_count} book(s) in {round(end - start, 2)} second(s)' )
def task(n, *args): print("Starting task %d" % n) time.sleep(random.random() * 5) print("Task %d done" % n) if random.random() < 0.1: raise RuntimeError("Oops...") return n * n if __name__ == "__main__": client = Client() # create a dependency chain tasks = {'task-0': (task, 1)} tasks.update( {'task-%d' % i: (task, i, 'task-%d' % (i - 1)) for i in range(1, 100)}) result = client.get(tasks, 'task-99', retries=10) print("Tasks done") print(result) print("Hit enter to quit") raw_input()