def frameworkMessage (self, driver, executorId, slaveId, message):
"""
Invoked when an executor sends a message. These messages are
best effort; do not expect a framework message to be
retransmitted in any reliable fashion.
"""
self.messagesReceived += 1
logging.info("Mesos Scheduler: slave %s executor %s", slaveId.value, executorId.value)
logging.info("message %d received: %s", self.messagesReceived, str(message))
if self.messagesReceived == self._n_workers:
if self.messagesReceived != self.messagesSent:
logging.critical("Mesos Scheduler: framework messages lost! sent %d received %d", self.messagesSent, self.messagesReceived)
sys.exit(1)
for exe in self._executors.values():
logging.debug(exe.report())
logging.info("all worker services launched and init tasks completed")
exe_info = self._executors.values()
worker_list = [ exe.get_shard_uri() for exe in exe_info ]
# run UnitOfWork orchestration via REST endpoints on the workers
fra = Framework(self._uow_name, self._prefix)
fra.set_worker_list(worker_list, exe_info)
time.sleep(1)
fra.orchestrate_uow()
# shutdown the Executors after the end of an algorithm run
driver.stop()
def frameworkMessage(self, driver, executorId, slaveId, message):
"""
Invoked when an executor sends a message. These messages are
best effort; do not expect a framework message to be
retransmitted in any reliable fashion.
"""
self.messagesReceived += 1
logging.info("Mesos Scheduler: slave %s executor %s", slaveId.value,
executorId.value)
logging.info("message %d received: %s", self.messagesReceived,
str(message))
if self.messagesReceived == self._n_workers:
if self.messagesReceived != self.messagesSent:
logging.critical(
"Mesos Scheduler: framework messages lost! sent %d received %d",
self.messagesSent, self.messagesReceived)
sys.exit(1)
for exe in self._executors.values():
logging.debug(exe.report())
logging.info(
"all worker services launched and init tasks completed")
exe_info = self._executors.values()
worker_list = [exe.get_shard_uri() for exe in exe_info]
# run UnitOfWork orchestration via REST endpoints on the workers
fra = Framework(self._uow_name, self._prefix)
fra.set_worker_list(worker_list, exe_info)
time.sleep(1)
fra.orchestrate_uow()
# shutdown the Executors after the end of an algorithm run
driver.stop()
args.cpu[0], args.mem[0])
MesosScheduler.stop_framework(driver)
except ImportError as e:
logging.critical("Python module 'mesos' has not been installed",
exc_info=True)
raise
elif args.slaves:
logging.info("%s: running a Framework in standalone mode", APP_NAME)
logging.info(" ...with slave(s) %s", args.slaves)
for x in opts:
logging.info(x)
# run UnitOfWork orchestration via REST endpoints on the workers
fra = Framework(args.uow[0], args.prefix[0])
fra.set_worker_list(args.slaves)
fra.orchestrate_uow()
elif args.port:
logging.info("%s: running a worker service on port %s", APP_NAME,
args.port[0])
try:
svc = Worker(port=int(args.port[0]))
svc.shard_start()
except KeyboardInterrupt:
pass
else:
logging.info("%s: running an Executor on an Apache Mesos slave",
# run Mesos driver to launch Framework and manage resource offers
driver = MesosScheduler.start_framework(master_uri, exe_path, args.workers[0], args.uow[0], args.prefix[0], args.cpu[0], args.mem[0])
MesosScheduler.stop_framework(driver)
except ImportError as e:
logging.critical("Python module 'mesos' has not been installed", exc_info=True)
raise
elif args.slaves:
logging.info("%s: running a Framework in standalone mode", APP_NAME)
logging.info(" ...with slave(s) %s", args.slaves)
for x in opts:
logging.info(x)
# run UnitOfWork orchestration via REST endpoints on the workers
fra = Framework(args.uow[0], args.prefix[0])
fra.set_worker_list(args.slaves)
fra.orchestrate_uow()
elif args.port:
logging.info("%s: running a worker service on port %s", APP_NAME, args.port[0])
try:
svc = Worker(port=int(args.port[0]))
svc.shard_start()
except KeyboardInterrupt:
pass
else:
logging.info("%s: running an Executor on an Apache Mesos slave", APP_NAME)
