class Listener(mapreduce_pb2_grpc.MapReduceWorkerServicer): def __init__(self, *args, **kwargs): self.rm = ResourceManager() self.kvstore = KeyValueStoreClient(self.rm.find_kvstore()) def Execute(self, task, context): print('Execute request. code_id:{}, chunk_id:{}, type:{}'.format( task.code_id, task.input_chunk_id, task.type)) log.info('Execute request. code_id:{}, chunk_id:{}, type:{}'.format( task.code_id, task.input_chunk_id, task.type)) # download code and chunk log.info('Downloadin chunk') workplace = os.path.join(INTERMEDIATE_OUTPUTS_DIR, task.output_doc_id) self.kvstore.download_directory(task.code_id, workplace, flatten=True) log.info('Downloading code') module_name = generateId() os.rename(os.path.join(workplace, 'task.py'), os.path.join(workplace, module_name + '.py')) try: sys.path.insert(1, workplace) py = __import__(module_name) output = [] if task.type == 'map': chunk = self.kvstore.read_chunk( task.input_chunk_id).decode(KV_STORE_ENCODING) for ch in py.mapper(task.input_doc_id, chunk): output.append(ch) self.kvstore.upload_bytes(task.output_dir_id, task.output_doc_id, pickle.dumps(output)) elif task.type == 'reduce': chunk = pickle.loads( self.kvstore.read_bytes(task.input_dir_id, task.input_doc_id)) for ch in py.reducer(chunk): output.append('{} {}'.format(ch[0], ch[1])) self.kvstore.upload_file_str(task.output_dir_id, task.output_doc_id, '\n'.join(output)) else: raise 'unknown operation' log.info('success') return mapreduce_pb2.ExecutionInfo(exec_id=task.output_dir_id, status='success') except BaseException as e: log.info('task failed {}'.format(e)) return mapreduce_pb2.ExecutionInfo(exec_id=task.output_dir_id, status='failed') finally: shutil.rmtree(workplace)
class Listener(mapreduce_pb2_grpc.MapReduceMasterServicer): def __init__(self, *args, **kwargs): log.info("Initializing MapReduceMaster") self.rm = ResourceManager() self.KV_STORE_HOST = self.rm.find_kvstore() self.kvstore = KeyValueStoreClient(self.KV_STORE_HOST) self.workers = Queue() self.workers_mutex = threading.Lock() log.info("Initializing MapReduceMaster is complete, KV_STORE_HOST: {}". format(self.KV_STORE_HOST)) def SubmitJob(self, job, context): # create new execution id exec_id = generateId() log.info('Received job: code_id:{} data_id:{}. Results will be stored under:{}'\ .format(job.code_id, job.data_id, exec_id)) yield mapreduce_pb2.ExecutionInfo(exec_id=exec_id, status='Received job') yield mapreduce_pb2.ExecutionInfo(exec_id=exec_id, status='Setting up workers') self.__launch_workers() yield mapreduce_pb2.ExecutionInfo( exec_id=exec_id, status='Setting up workers is complete') # get all chunk ids from database log.info('Downloading chunk metadata from KeyValueStore') chunks = self.kvstore.get_chunk_metadata(job.data_id) worker_threads = [] # send chunk_id and code_id to available worker mapper_outputs = [] for idx, chunk in enumerate(chunks): worker = self.__getworker() status = 'map_task {}/{} --> {}'.format(idx + 1, len(chunks), worker) log.info(status) yield mapreduce_pb2.ExecutionInfo(exec_id=exec_id, status=status) # create new doc_id to store intermediate mapper output mapper_output_doc_id = generateId() mapper_outputs.append(mapper_output_doc_id) task = mapreduce_pb2.Task(code_id=job.code_id, type='map', \ input_dir_id=exec_id, input_doc_id=chunk[1], input_chunk_id=chunk[3], \ output_dir_id=exec_id, output_doc_id=mapper_output_doc_id) log.debug(task_str(task)) log.debug(worker_str(worker)) # start task in thread wthread = threading.Thread(target=self.__execute_chunk, args=( worker, task, ), daemon=True) wthread.start() worker_threads.append(wthread) # wait for all threads to finish for wthread in worker_threads: wthread.join() status = 'All map_tasks finished' log.info(status) yield mapreduce_pb2.ExecutionInfo(exec_id=exec_id, status=status) shuffled_output = {} for doc_id in mapper_outputs: data = self.kvstore.read_bytes(exec_id, doc_id) if data: output = pickle.loads(data) for entry in output: if entry[0] not in shuffled_output: shuffled_output[entry[0]] = [] shuffled_output[entry[0]].append(entry[1]) shuffled_output_doc_id = generateId() self.kvstore.upload_bytes(exec_id, shuffled_output_doc_id, pickle.dumps(shuffled_output)) status = 'Shuffling and sorting of mapper outputs is complete.' log.info(status) yield mapreduce_pb2.ExecutionInfo(exec_id=exec_id, status=status) worker = self.__getworker() reducer_output_doc_id = 'output' task = mapreduce_pb2.Task(code_id=job.code_id, type='reduce', \ input_dir_id=exec_id, input_doc_id=shuffled_output_doc_id, input_chunk_id='', \ output_dir_id=exec_id, output_doc_id=reducer_output_doc_id) # start task in thread wthread = threading.Thread(target=self.__execute_chunk, args=( worker, task, ), daemon=True) wthread.start() status = 'reduce_task --> {}'.format(worker) log.info(status) yield mapreduce_pb2.ExecutionInfo(exec_id=exec_id, status=status) wthread.join() status = 'reduce_task finished.' log.info(status) yield mapreduce_pb2.ExecutionInfo(exec_id=exec_id, status=status) status = 'Success' log.info(status) yield mapreduce_pb2.ExecutionInfo(exec_id=exec_id, status=status) yield mapreduce_pb2.ExecutionInfo(exec_id=exec_id, status='Cleaning up workers') self.rm.destroy_workers() yield mapreduce_pb2.ExecutionInfo( exec_id=exec_id, status='Cleaning up workers is complete') def __getworker(self): log.info("Waiting for worker") worker = None # loop until we successfully acquire a worker while not worker: # check if worker is still available self.workers_mutex.acquire() if not self.workers.empty(): worker = self.workers.get() else: log.info('all workers busy, wait..') self.workers_mutex.release() if not worker: time.sleep(1) log.info("Got one worker") return worker def __execute_chunk(self, worker, task): try: # assign chunk to worker with grpc.insecure_channel("{}:{}".format(worker[0], worker[1])) as channel: stub = mapreduce_pb2_grpc.MapReduceWorkerStub(channel) log.debug('Completed ' + task_str(task)) return stub.Execute(task) except BaseException as e: print(e) log.error('Error executing ' + task_str(task) + ', ' + str(e)) finally: # return the work to the pool self.workers_mutex.acquire() self.workers.put(worker) self.workers_mutex.release() def __launch_workers(self): log.info("Creating workers") worker_list = self.rm.create_workers(NUM_WORKERS) log.info("Workers created" + str(worker_list)) shuffle(worker_list) self.workers = Queue() for worker in worker_list: for _ in range(TASKS_PER_WORKER): self.workers.put(worker)