def _sendFrameworkMessage(self, driver):
message = None
while True:
# The psutil documentation recommends that we ignore the value returned by the first
# invocation of cpu_percent(). However, we do want to send a sign of life early after
# starting (e.g. to unblock the provisioner waiting for an instance to come up) so
# the first message we send omits the load info.
if message is None:
message = Expando(address=self.address)
psutil.cpu_percent()
else:
message.nodeInfo = dict(cores=float(psutil.cpu_percent()) * .01,
memory=float(psutil.virtual_memory().percent) * .01,
workers=len(self.runningTasks))
driver.sendFrameworkMessage(repr(message))
# Prevent workers launched together from repeatedly hitting the leader at the same time
sleep(random.randint(45, 75))
def _sendFrameworkMessage(self, driver):
message = None
while True:
# The psutil documentation recommends that we ignore the value returned by the first
# invocation of cpu_percent(). However, we do want to send a sign of life early after
# starting (e.g. to unblock the provisioner waiting for an instance to come up) so
# the first message we send omits the load info.
if message is None:
message = Expando(address=self.address)
psutil.cpu_percent()
else:
message.nodeInfo = dict(cores=float(psutil.cpu_percent()) * .01,
memory=float(psutil.virtual_memory().percent) * .01,
workers=len(self.runningTasks))
driver.sendFrameworkMessage(repr(message))
# Prevent workers launched together from repeatedly hitting the leader at the same time
sleep(random.randint(45, 75))
def frameworkMessage(self, driver, executorId, slaveId, message):
"""
Invoked when an executor sends a message.
"""
message = ast.literal_eval(message)
assert isinstance(message, dict)
# Handle the mandatory fields of a message
nodeAddress = message.pop('address')
executor = self.executors.get(nodeAddress)
if executor is None or executor.slaveId != slaveId:
executor = Expando(nodeAddress=nodeAddress, slaveId=slaveId, nodeInfo=None)
self.executors[nodeAddress] = executor
executor.lastSeen = time.time()
# Handle optional message fields
for k, v in message.iteritems():
if k == 'nodeInfo':
assert isinstance(v, dict)
executor.nodeInfo = NodeInfo(**v)
else:
raise RuntimeError("Unknown message field '%s'." % k)
def frameworkMessage(self, driver, executorId, slaveId, message):
"""
Invoked when an executor sends a message.
"""
message = ast.literal_eval(message)
assert isinstance(message, dict)
# Handle the mandatory fields of a message
nodeAddress = message.pop('address')
executor = self.executors.get(nodeAddress)
if executor is None or executor.slaveId != slaveId:
executor = Expando(nodeAddress=nodeAddress,
slaveId=slaveId,
nodeInfo=None)
self.executors[nodeAddress] = executor
executor.lastSeen = time.time()
# Handle optional message fields
for k, v in message.iteritems():
if k == 'nodeInfo':
assert isinstance(v, dict)
executor.nodeInfo = NodeInfo(**v)
else:
raise RuntimeError("Unknown message field '%s'." % k)
