def __init__(self):
"""Initialize queue to empty elements and create a communication
object."""
self.movable = deque()
self.ready = deque()
self.inprogress = {}
self.socket = Communicator()
if scoop.SIZE == 1 and not scoop.CONFIGURATION.get('headless', False):
self.lowwatermark = float("inf")
self.highwatermark = float("inf")
else:
# TODO: Make it dependent on the network latency
self.lowwatermark = 0.01
self.highwatermark = 0.01
def __init__(self):
"""Initialize queue to empty elements and create a communication
object."""
self.movable = deque()
self.ready = deque()
self.inprogress = set()
self.socket = Communicator()
self.request_in_process = False
if scoop.SIZE == 1 and not scoop.CONFIGURATION.get('headless', False):
self.lowwatermark = float("inf")
self.highwatermark = float("inf")
else:
# TODO: Make it dependent on the network latency
self.lowwatermark = 0.01
self.highwatermark = 0.01
class FutureQueue(object):
"""This class encapsulates a queue of futures that are pending execution.
Within this class lies the entry points for future communications."""
def __init__(self):
"""Initialize queue to empty elements and create a communication
object."""
self.movable = deque()
self.ready = deque()
self.inprogress = set()
self.socket = Communicator()
self.lastStatus = 0.0
if scoop.SIZE == 1 and not scoop.CONFIGURATION.get('headless', False):
self.lowwatermark = float("inf")
self.highwatermark = float("inf")
else:
# TODO: Make it dependent on the network latency
self.lowwatermark = 0.01
self.highwatermark = 0.01
def __del__(self):
"""Destructor. Ensures Communicator is correctly discarted."""
self.shutdown()
def __iter__(self):
"""Iterates over the selectable (cancellable) elements of the queue."""
return itertools.chain(self.movable, self.ready)
def __len__(self):
"""Returns the length of the queue, meaning the sum of it's elements
lengths."""
return len(self.movable) + len(self.ready)
def timelen(self, queue_):
stats = scoop._control.execStats
times = Counter(hash(f.callable) for f in queue_)
return sum(stats[f].median() * occur for f, occur in times.items())
def append(self, future):
"""Append a future to the queue."""
if future._ended() and future.index is None:
self.inprogress.add(future)
elif future._ended() and future.index is not None:
self.ready.append(future)
elif future.greenlet is not None:
self.inprogress.add(future)
else:
self.movable.append(future)
# Send the oldest future in the movable deque until under the hwm
over_hwm = self.timelen(self.movable) > self.highwatermark
while over_hwm and len(self.movable) > 1:
sending_future = self.movable.popleft()
if sending_future.id[0] != scoop.worker:
sending_future._delete()
self.socket.sendFuture(sending_future)
over_hwm = self.timelen(self.movable) > self.highwatermark
def askForPreviousFutures(self):
"""Request a status for every future to the broker."""
# Don't request it too often (otherwise it ping-pongs because)
# the broker answer triggers the _poll of pop()
if time.time() < self.lastStatus + POLLING_TIME / 1000:
return
self.lastStatus = time.time()
for future in scoop._control.futureDict.values():
# Skip the root future
if scoop.IS_ORIGIN and future.id == (scoop.worker, 0):
continue
if future not in self.inprogress:
self.socket.sendStatusRequest(future)
def pop(self):
"""Pop the next future from the queue;
in progress futures have priority over those that have not yet started;
higher level futures have priority over lower level ones; """
self.updateQueue()
# If our buffer is underflowing, request more Futures
if self.timelen(self) < self.lowwatermark:
self.requestFuture()
# If an unmovable Future is ready to be executed, return it
if len(self.ready) != 0:
return self.ready.popleft()
# Then, use Futures in the movable queue
elif len(self.movable) != 0:
return self.movable.popleft()
else:
# Otherwise, block until a new task arrives
self.lastStatus = time.time()
while len(self) == 0:
# Block until message arrives
self.askForPreviousFutures()
self.socket._poll(POLLING_TIME)
self.updateQueue()
if len(self.ready) != 0:
return self.ready.popleft()
elif len(self.movable) != 0:
return self.movable.popleft()
def flush(self):
"""Empty the local queue and send its elements to be executed remotely.
"""
for elem in self:
if elem.id[0] != scoop.worker:
elem._delete()
self.socket.sendFuture(elem)
self.ready.clear()
self.movable.clear()
def requestFuture(self):
"""Request futures from the broker"""
self.socket.sendRequest()
def updateQueue(self):
"""Process inbound communication buffer.
Updates the local queue with elements from the broker."""
for future in self.socket.recvFuture():
if future._ended():
# If the answer is coming back, update its entry
try:
thisFuture = scoop._control.futureDict[future.id]
except KeyError:
# Already received?
scoop.logger.warn('{0}: Received an unexpected future: '
'{1}'.format(scoop.worker, future.id))
continue
thisFuture.resultValue = future.resultValue
thisFuture.exceptionValue = future.exceptionValue
thisFuture.executor = future.executor
thisFuture.isDone = future.isDone
# Execute standard callbacks here (on parent)
thisFuture._execute_callbacks(CallbackType.standard)
self.append(thisFuture)
future._delete()
elif future.id not in scoop._control.futureDict:
scoop._control.futureDict[future.id] = future
self.append(scoop._control.futureDict[future.id])
else:
self.append(scoop._control.futureDict[future.id])
def remove(self, future):
"""Remove a future from the queue. The future must be cancellable or
this method will raise a ValueError."""
self.movable.remove(future)
def sendResult(self, future):
"""Send back results to broker for distribution to parent task."""
# Greenlets cannot be pickled
future.greenlet = None
assert future._ended(), "The results are not valid"
self.socket.sendResult(future)
def shutdown(self):
"""Shutdown the ressources used by the queue"""
self.socket.shutdown()
if scoop:
if scoop.DEBUG:
from scoop import _debug
_debug.writeWorkerDebug(
scoop._control.debug_stats,
scoop._control.QueueLength,
)
class FutureQueue(object):
"""This class encapsulates a queue of futures that are pending execution.
Within this class lies the entry points for future communications."""
def __init__(self):
"""Initialize queue to empty elements and create a communication
object."""
self.movable = deque()
self.ready = deque()
self.inprogress = set()
self.socket = Communicator()
self.request_in_process = False
if scoop.SIZE == 1 and not scoop.CONFIGURATION.get('headless', False):
self.lowwatermark = float("inf")
self.highwatermark = float("inf")
else:
# TODO: Make it dependent on the network latency
self.lowwatermark = 0.01
self.highwatermark = 0.01
def __del__(self):
"""Destructor. Ensures Communicator is correctly discarted."""
self.shutdown()
def __iter__(self):
"""Iterates over the selectable (cancellable) elements of the queue."""
return itertools.chain(self.movable, self.ready)
def __len__(self):
"""Returns the length of the queue, meaning the sum of it's elements
lengths."""
return len(self.movable) + len(self.ready)
def timelen(self, queue_):
stats = scoop._control.execStats
times = Counter(hash(f.callable) for f in queue_)
return sum(stats[f].median() * occur for f, occur in times.items())
def append_ready(self, future):
"""
This appends a ready future to the queue.
NOTE: A Future is ready iff it has completed execution and been
processed by the worker that generated it
"""
if future.isReady:
self.ready.append(future)
else:
raise ValueError(
("The future id {} has not been assimilated"
" before adding, on worker: {}").format(future.id,
scoop.worker))
def append_init(self, future):
"""
This appends a movable future to the queue FOR THE FIRST TIME.
NOTE: This is different from append_movable in that all the futures
are not actually appended but are sent to the broker.
"""
if future.greenlet is None and not future.isDone and future.id[0] == scoop.worker:
self.socket.sendFuture(future)
else:
raise ValueError((
"The future id {} being added to queue initially is not "
" movable before adding, on worker: {}").format(future.id,
scoop.worker))
def append_movable(self, future):
"""
This appends a movable future to the queue.
NOTE: A Future is movable if it hasn't yet begun execution. This is
characterized by the lack of a greenlet and not having completed. Also note
that this function is only called when appending a future retrieved from the
broker. to append a newly spawned future, use `FutureQueue.append_init`
"""
if future.greenlet is None and not future.isDone:
self.movable.append(future)
assert len(self.movable) == 1, "movable size isnt adding up"
else:
raise ValueError((
"The future id {} being added to movable queue is not "
" movable before adding, on worker: {}").format(future.id,
scoop.worker))
def pop(self):
"""Pop the next future from the queue;
ready futures have priority over those that have not yet started;
It is ASSUMED that any queue popped from the movable queue, identifiable
by _ended() == False (Note that self.inprogress is never 'popped') is
going to be executed and hence will be added to the inprogress set of
execQueue."""
# Check if queue is empty
while len(self) == 0:
# If so, Block until message arrives. Only send future request once (to
# ensure FCFS). This has the following potential issue. If a node
# disconnects and reconnects and is considered by the broker to be lost,
# there is a possibility that it has been removed from the brokers list
# of assignable workers, in which case, this worker will forever be
# stuck in this loop. However, this is a problem that is expected to
# NEVER happen and therefore we leave it be. Currently, I have added
# some code that can be used to protect against this (see
# FutureQueue.checkRequestStatus, REQUEST_STATUS_REQUEST and related)
if not self.request_in_process:
self.requestFuture()
self.socket._poll(POLLING_TIME)
self.updateQueue()
if len(self.ready) != 0:
return self.ready.popleft()
elif len(self.movable) != 0:
self.inprogress.add(self.movable[0])
return self.movable.popleft()
def flush(self):
"""Empty the local queue and send its elements to be executed remotely.
"""
for elem in self:
if elem.id[0] != scoop.worker:
scoop._control.delFuture(elem)
self.socket.sendFuture(elem)
self.ready.clear()
self.movable.clear()
def requestFuture(self):
"""Request futures from the broker"""
self.socket.sendRequest()
self.request_in_process = True
def updateQueue(self):
"""Process inbound communication buffer.
Updates the local queue with elements from the broker.
Note that the broker only sends either non-executed (movable)
futures, or completed futures"""
for incoming_msg in self.socket.recvIncoming():
incoming_msg_categ = incoming_msg[0]
incoming_msg_value = incoming_msg[1]
if incoming_msg_categ == REPLY:
future = incoming_msg_value
# If the answer is coming back, update its entry
try:
thisFuture = scoop._control.futureDict[future.id]
except KeyError:
# Already received?
scoop.logger.warn('{0}: Received an unexpected future: '
'{1}'.format(scoop.worker, future.id))
return
thisFuture.resultValue = future.resultValue
thisFuture.exceptionValue = future.exceptionValue
thisFuture.executor = future.executor
thisFuture.isDone = future.isDone
self.finalizeReturnedFuture(thisFuture)
elif incoming_msg_categ == TASK:
future = incoming_msg_value
if future.id not in scoop._control.futureDict:
# This is the case where the worker is executing a remotely
# generated future
scoop._control.futureDict[future.id] = future
self.append_movable(scoop._control.futureDict[future.id])
else:
# This is the case where the worker is executing a locally
# generated future
self.append_movable(scoop._control.futureDict[future.id])
if len(self.movable) > 0:
# This means that a future has been returned corresponding to the
# future request
self.request_in_process = False
elif incoming_msg_categ == RESEND_FUTURE:
future_id = incoming_msg_value
try:
scoop.logger.warning(
"Lost track of future {0}. Resending it..."
"".format(scoop._control.futureDict[future_id])
)
self.socket.sendFuture(scoop._control.futureDict[future_id])
except KeyError:
# Future was received and processed meanwhile
scoop.logger.warning(
"Asked to resend unexpected future id {0}. future not found"
" (likely received and processed in the meanwhile)"
"".format(future_id)
)
else:
assert False, "Unrecognized incoming message"
def finalizeReturnedFuture(self, future):
"""Finalize a future that was generated here and executed remotely.
"""
if not (future.executor[0] != scoop.worker == future.id[0]
and future.isDone):
raise UnrecognizedFuture(("The future ID {0} was not executed"
" remotely and returned, worker: {1}")
.format(future.id, scoop.worker))
# Execute standard callbacks here (on parent)
future._execute_callbacks(CallbackType.standard)
scoop._control.delFuture(future)
future.isReady = True
self.append_ready(future)
def finalizeFuture(self, future):
"""Finalize an ended future, this does the following:
1. The future is checked to see if it was inprogress or not on this thread,
2. All references to the future are removed
3. The future is added to the ready queue of the parent process
NOTE: After this function, do not continue to process this future, pick
a new one from the queue
"""
if not (future in self.inprogress and future.isDone):
raise UnrecognizedFuture(
("The future ID {0} was not in progress on worker"
" {1}, finalize is undefined ").format(future.id, scoop.worker))
scoop._control.delFuture(future)
self.inprogress.remove(future)
if future.id[0] == scoop.worker:
future.isReady = True
self.append_ready(future)
else:
self.sendResult(future)
def sendReadyStatus(self, future):
"""This should only be called after the future has been finalized on the worker.
"""
self.socket.sendReadyStatus(future)
def remove(self, future):
"""Remove a future from the queue. The future must be cancellable or
this method will raise a ValueError."""
self.movable.remove(future)
def sendResult(self, future):
"""Send back results to broker for distribution to parent task."""
# Greenlets cannot be pickled
future.greenlet = None
assert future._ended(), "The results are not valid"
self.socket.sendResult(future)
def shutdown(self):
"""Shutdown the ressources used by the queue"""
self.socket.shutdown()
if scoop:
if scoop.DEBUG:
from scoop import _debug
_debug.writeWorkerDebug(
scoop._control.debug_stats,
scoop._control.QueueLength,
)
class FutureQueue(object):
"""This class encapsulates a queue of futures that are pending execution.
Within this class lies the entry points for future communications."""
def __init__(self):
"""Initialize queue to empty elements and create a communication
object."""
self.movable = deque()
self.ready = deque()
self.inprogress = set()
self.socket = Communicator()
self.lastStatus = 0.0
if scoop.SIZE == 1 and not scoop.CONFIGURATION.get('headless', False):
self.lowwatermark = float("inf")
self.highwatermark = float("inf")
else:
# TODO: Make it dependent on the network latency
self.lowwatermark = 0.01
self.highwatermark = 0.01
def __del__(self):
"""Destructor. Ensures Communicator is correctly discarted."""
self.shutdown()
def __iter__(self):
"""Iterates over the selectable (cancellable) elements of the queue."""
return itertools.chain(self.movable, self.ready)
def __len__(self):
"""Returns the length of the queue, meaning the sum of it's elements
lengths."""
return len(self.movable) + len(self.ready)
def timelen(self, queue_):
stats = scoop._control.execStats
times = Counter(hash(f.callable) for f in queue_)
return sum(stats[f].median() * occur for f, occur in times.items())
def append(self, future):
"""Append a future to the queue."""
if future._ended() and future.index is None:
self.inprogress.add(future)
elif future._ended() and future.index is not None:
self.ready.append(future)
elif future.greenlet is not None:
self.inprogress.add(future)
else:
self.movable.append(future)
# Send the oldest future in the movable deque until under the hwm
over_hwm = self.timelen(self.movable) > self.highwatermark
while over_hwm and len(self.movable) > 1:
sending_future = self.movable.popleft()
if sending_future.id[0] != scoop.worker:
sending_future._delete()
self.socket.sendFuture(sending_future)
over_hwm = self.timelen(self.movable) > self.highwatermark
def askForPreviousFutures(self):
"""Request a status for every future to the broker."""
# Don't request it too often (otherwise it ping-pongs because)
# the broker answer triggers the _poll of pop()
if time.time() < self.lastStatus + POLLING_TIME / 1000:
return
self.lastStatus = time.time()
for future in scoop._control.futureDict.values():
# Skip the root future
if scoop.IS_ORIGIN and future.id == (scoop.worker, 0):
continue
if future not in self.inprogress:
self.socket.sendStatusRequest(future)
def pop(self):
"""Pop the next future from the queue;
in progress futures have priority over those that have not yet started;
higher level futures have priority over lower level ones; """
self.updateQueue()
# If our buffer is underflowing, request more Futures
if self.timelen(self) < self.lowwatermark:
self.requestFuture()
# If an unmovable Future is ready to be executed, return it
if len(self.ready) != 0:
return self.ready.popleft()
# Then, use Futures in the movable queue
elif len(self.movable) != 0:
return self.movable.popleft()
else:
# Otherwise, block until a new task arrives
self.lastStatus = time.time()
while len(self) == 0:
# Block until message arrives
self.askForPreviousFutures()
self.socket._poll(POLLING_TIME)
self.updateQueue()
if len(self.ready) != 0:
return self.ready.popleft()
elif len(self.movable) != 0:
return self.movable.popleft()
def flush(self):
"""Empty the local queue and send its elements to be executed remotely.
"""
for elem in self:
if elem.id[0] != scoop.worker:
elem._delete()
self.socket.sendFuture(elem)
self.ready.clear()
self.movable.clear()
def requestFuture(self):
"""Request futures from the broker"""
self.socket.sendRequest()
def updateQueue(self):
"""Process inbound communication buffer.
Updates the local queue with elements from the broker."""
for future in self.socket.recvFuture():
if future._ended():
# If the answer is coming back, update its entry
try:
thisFuture = scoop._control.futureDict[future.id]
except KeyError:
# Already received?
scoop.logger.warn('{0}: Received an unexpected future: '
'{1}'.format(scoop.worker, future.id))
continue
thisFuture.resultValue = future.resultValue
thisFuture.exceptionValue = future.exceptionValue
thisFuture.executor = future.executor
thisFuture.isDone = future.isDone
# Execute standard callbacks here (on parent)
thisFuture._execute_callbacks(CallbackType.standard)
self.append(thisFuture)
future._delete()
elif future.id not in scoop._control.futureDict:
scoop._control.futureDict[future.id] = future
self.append(scoop._control.futureDict[future.id])
else:
self.append(scoop._control.futureDict[future.id])
def remove(self, future):
"""Remove a future from the queue. The future must be cancellable or
this method will raise a ValueError."""
self.movable.remove(future)
def sendResult(self, future):
"""Send back results to broker for distribution to parent task."""
# Greenlets cannot be pickled
future.greenlet = None
assert future._ended(), "The results are not valid"
self.socket.sendResult(future)
def shutdown(self):
"""Shutdown the ressources used by the queue"""
self.socket.shutdown()
if scoop:
if scoop.DEBUG:
from scoop import _debug
_debug.writeWorkerDebug(
scoop._control.debug_stats,
scoop._control.QueueLength,
)
