def _sendFrameworkMessage(self, driver): message = None while True: # The psutil documentation recommends that we ignore the value returned by the first # invocation of cpu_percent(). However, we do want to send a sign of life early after # starting (e.g. to unblock the provisioner waiting for an instance to come up) so # the first message we send omits the load info. if message is None: message = Expando(address=self.address) psutil.cpu_percent() else: message.nodeInfo = dict(cores=float(psutil.cpu_percent()) * .01, memory=float(psutil.virtual_memory().percent) * .01, workers=len(self.runningTasks)) driver.sendFrameworkMessage(repr(message)) # Prevent workers launched together from repeatedly hitting the leader at the same time sleep(random.randint(45, 75))
def frameworkMessage(self, driver, executorId, slaveId, message): """ Invoked when an executor sends a message. """ message = ast.literal_eval(message) assert isinstance(message, dict) # Handle the mandatory fields of a message nodeAddress = message.pop('address') executor = self.executors.get(nodeAddress) if executor is None or executor.slaveId != slaveId: executor = Expando(nodeAddress=nodeAddress, slaveId=slaveId, nodeInfo=None) self.executors[nodeAddress] = executor executor.lastSeen = time.time() # Handle optional message fields for k, v in message.iteritems(): if k == 'nodeInfo': assert isinstance(v, dict) executor.nodeInfo = NodeInfo(**v) else: raise RuntimeError("Unknown message field '%s'." % k)