class Master(object): def __init__(self, task_loader, conf): self.lock = threading.Lock() self.workerManager = WorkerManager(conf) self.taskManager = TaskManager(task_loader) self.running_tasks = {} self.conf = conf self.load_tasks() def get_status(self): return { 'total_workers': self.workerManager.get_workers(), 'tasks': self.taskManager.get_tasks_stats(), 'idle_workers': self.workerManager.get_idle_workers() } def clean_death_workers(self): '''定期检查worker的心跳信息,及时清除死亡worker''' workers, tasks = self.workerManager.clean_death_workers() logging.info("death workers:%s; relatedTasks:%s", workers, tasks) for task in tasks: self.taskManager.fail_task(task.uuid, TaskStatus.notReturned) return workers def register_worker(self, worker): '''注册作业节点''' logging.info("%s come in", worker) status = "OK" if worker is not None: self.workerManager.add_worker(worker) else: status = "Invalid" # logging.info(self.workerManager.get_workers()) return status def remove_worker(self, worker): status = "OK" if worker is None: status = "Invalid" return status identifier = worker.get_uuid() w, tasks = self.workerManager.remove_worker(identifier) for task in tasks: self.taskManager.fail_task(task.get_uuid(), TaskStatus.notReturned) if w is None: status = "NOT EXISTS" return status def task_complete(self, worker, taskResult): '''worker完成一个作业,返回作业统计信息,worker重新归于队列''' self.workerManager.finish_task(worker, taskResult) self.workerManager.update_worker(worker) if taskResult.is_success(): self.taskManager.finish_task(taskResult.get_task_uuid()) else: self.taskManager.fail_task(taskResult.get_task_uuid(), TaskStatus.failedToExecute) return True def heartbeat(self, worker): '''收到心跳信息,更新该工作节点的信息''' self.workerManager.update_worker(worker) return True def lookup_spider(self, spider): pass def load_tasks(self): self.taskManager.load_tasks() def schedule_next(self): logging.info('tasks: %s', self.taskManager.get_tasks_stats()) task = self.taskManager.next_task() worker = self.workerManager.next_worker() self.workerManager.assign_task(worker, task) try: proxy = common.RPCServerProxy.get_proxy(worker) proxy.assign_task(task) except Exception, e: traceback.print_exc() self.remove_worker(worker)
class Master(object): def __init__(self, task_loader, conf): self.lock = threading.Lock() self.workerManager = WorkerManager(conf) self.taskManager = TaskManager(task_loader) self.running_tasks = {} self.conf = conf self.load_tasks() def get_status(self): return { 'total_workers': self.workerManager.get_workers(), 'tasks': self.taskManager.get_tasks_stats(), 'idle_workers': self.workerManager.get_idle_workers() } def clean_death_workers(self): '''定期检查worker的心跳信息,及时清除死亡worker''' workers,tasks = self.workerManager.clean_death_workers() logging.info("death workers:%s; relatedTasks:%s", workers, tasks) for task in tasks: self.taskManager.fail_task(task.uuid, TaskStatus.notReturned) return workers def register_worker(self, worker): '''注册作业节点''' logging.info("%s come in", worker) status = "OK" if worker is not None: self.workerManager.add_worker(worker) else: status = "Invalid" # logging.info(self.workerManager.get_workers()) return status def remove_worker(self, worker): status = "OK" if worker is None: status = "Invalid" return status identifier = worker.get_uuid() w, tasks = self.workerManager.remove_worker(identifier) for task in tasks: self.taskManager.fail_task(task.get_uuid(), TaskStatus.notReturned) if w is None: status = "NOT EXISTS" return status def task_complete(self, worker, taskResult): '''worker完成一个作业,返回作业统计信息,worker重新归于队列''' self.workerManager.finish_task(worker, taskResult) self.workerManager.update_worker(worker) if taskResult.is_success(): self.taskManager.finish_task(taskResult.get_task_uuid()) else: self.taskManager.fail_task(taskResult.get_task_uuid(), TaskStatus.failedToExecute) return True def heartbeat(self, worker): '''收到心跳信息,更新该工作节点的信息''' self.workerManager.update_worker(worker) return True def lookup_spider(self, spider): pass def load_tasks(self): self.taskManager.load_tasks() def schedule_next(self): logging.info('tasks: %s',self.taskManager.get_tasks_stats()) task = self.taskManager.next_task() worker = self.workerManager.next_worker() self.workerManager.assign_task(worker, task) try: proxy = common.RPCServerProxy.get_proxy(worker) proxy.assign_task(task) except Exception,e: traceback.print_exc() self.remove_worker(worker)