def frameworkMessage (self, driver, executorId, slaveId, message): """ Invoked when an executor sends a message. These messages are best effort; do not expect a framework message to be retransmitted in any reliable fashion. """ self.messagesReceived += 1 logging.info("Mesos Scheduler: slave %s executor %s", slaveId.value, executorId.value) logging.info("message %d received: %s", self.messagesReceived, str(message)) if self.messagesReceived == self._n_workers: if self.messagesReceived != self.messagesSent: logging.critical("Mesos Scheduler: framework messages lost! sent %d received %d", self.messagesSent, self.messagesReceived) sys.exit(1) for exe in self._executors.values(): logging.debug(exe.report()) logging.info("all worker services launched and init tasks completed") exe_info = self._executors.values() worker_list = [ exe.get_shard_uri() for exe in exe_info ] # run UnitOfWork orchestration via REST endpoints on the workers fra = Framework(self._uow_name, self._prefix) fra.set_worker_list(worker_list, exe_info) time.sleep(1) fra.orchestrate_uow() # shutdown the Executors after the end of an algorithm run driver.stop()
def frameworkMessage(self, driver, executorId, slaveId, message): """ Invoked when an executor sends a message. These messages are best effort; do not expect a framework message to be retransmitted in any reliable fashion. """ self.messagesReceived += 1 logging.info("Mesos Scheduler: slave %s executor %s", slaveId.value, executorId.value) logging.info("message %d received: %s", self.messagesReceived, str(message)) if self.messagesReceived == self._n_workers: if self.messagesReceived != self.messagesSent: logging.critical( "Mesos Scheduler: framework messages lost! sent %d received %d", self.messagesSent, self.messagesReceived) sys.exit(1) for exe in self._executors.values(): logging.debug(exe.report()) logging.info( "all worker services launched and init tasks completed") exe_info = self._executors.values() worker_list = [exe.get_shard_uri() for exe in exe_info] # run UnitOfWork orchestration via REST endpoints on the workers fra = Framework(self._uow_name, self._prefix) fra.set_worker_list(worker_list, exe_info) time.sleep(1) fra.orchestrate_uow() # shutdown the Executors after the end of an algorithm run driver.stop()
args.cpu[0], args.mem[0]) MesosScheduler.stop_framework(driver) except ImportError as e: logging.critical("Python module 'mesos' has not been installed", exc_info=True) raise elif args.slaves: logging.info("%s: running a Framework in standalone mode", APP_NAME) logging.info(" ...with slave(s) %s", args.slaves) for x in opts: logging.info(x) # run UnitOfWork orchestration via REST endpoints on the workers fra = Framework(args.uow[0], args.prefix[0]) fra.set_worker_list(args.slaves) fra.orchestrate_uow() elif args.port: logging.info("%s: running a worker service on port %s", APP_NAME, args.port[0]) try: svc = Worker(port=int(args.port[0])) svc.shard_start() except KeyboardInterrupt: pass else: logging.info("%s: running an Executor on an Apache Mesos slave",
# run Mesos driver to launch Framework and manage resource offers driver = MesosScheduler.start_framework(master_uri, exe_path, args.workers[0], args.uow[0], args.prefix[0], args.cpu[0], args.mem[0]) MesosScheduler.stop_framework(driver) except ImportError as e: logging.critical("Python module 'mesos' has not been installed", exc_info=True) raise elif args.slaves: logging.info("%s: running a Framework in standalone mode", APP_NAME) logging.info(" ...with slave(s) %s", args.slaves) for x in opts: logging.info(x) # run UnitOfWork orchestration via REST endpoints on the workers fra = Framework(args.uow[0], args.prefix[0]) fra.set_worker_list(args.slaves) fra.orchestrate_uow() elif args.port: logging.info("%s: running a worker service on port %s", APP_NAME, args.port[0]) try: svc = Worker(port=int(args.port[0])) svc.shard_start() except KeyboardInterrupt: pass else: logging.info("%s: running an Executor on an Apache Mesos slave", APP_NAME)