def _startup(rootFuture, *args, **kargs):
"""Initializes the SCOOP environment.
:param rootFuture: Any callable object (function or class object with *__call__*
method); this object will be called once and allows the use of parallel
calls inside this object.
:param args: A tuple of positional arguments that will be passed to the
callable object.
:param kargs: A dictionary of additional keyword arguments that will be
passed to the callable object.
:returns: The result of the root Future.
Be sure to launch your root Future using this method."""
import greenlet
global _controller
_controller = greenlet.greenlet(control.runController)
try:
result = _controller.switch(rootFuture, *args, **kargs)
except scoop._comm.Shutdown:
result = None
control.execQueue.shutdown()
if scoop.DEBUG:
from scoop import _debug
_debug.writeWorkerDebug(control.debug_stats, control.QueueLength)
return result
def shutdown(self):
"""Shutdown the ressources used by the queue"""
self.socket.shutdown()
if scoop:
if scoop.DEBUG:
from scoop import _debug
_debug.writeWorkerDebug(
scoop._control.debug_stats,
scoop._control.QueueLength,
)
def shutdown(self):
"""Shutdown the ressources used by the queue"""
self.socket.shutdown()
if scoop:
if scoop.DEBUG:
from scoop import _debug
_debug.writeWorkerDebug(
scoop._control.debug_stats,
scoop._control.QueueLength,
)
def runController(callable_, *args, **kargs):
"""Callable greenlet implementing controller logic."""
global execQueue
# initialize and run root future
rootId = FutureId(-1, 0)
# initialise queue
if execQueue is None:
execQueue = FutureQueue()
sys.excepthook = advertiseBrokerWorkerDown
# TODO: Make that a function
# Wait until we received the main module if we are a headless slave
headless = scoop.CONFIGURATION.get("headless", False)
if not scoop.MAIN_MODULE:
# If we're not the origin and still don't have our main_module,
# wait for it and then import it as module __main___
main = scoop.shared.getConst('__MAIN_MODULE__',
timeout=float('inf'))
directory_name = tempfile.mkdtemp()
os.chdir(directory_name)
scoop.MAIN_MODULE = main.writeFile(directory_name)
from .bootstrap.__main__ import Bootstrap as SCOOPBootstrap
newModule = SCOOPBootstrap.setupEnvironment()
sys.modules['__main__'] = newModule
elif scoop.IS_ORIGIN and headless and scoop.MAIN_MODULE:
# We're the origin, share our main_module
scoop.shared.setConst(
__MAIN_MODULE__=scoop.encapsulation.ExternalEncapsulation(
scoop.MAIN_MODULE, ))
# TODO: use modulefinder to share every local dependency of
# main module
# launch future if origin or try to pickup a future if slave worker
if scoop.IS_ORIGIN:
future = Future(rootId, callable_, *args, **kargs)
else:
future = execQueue.pop()
future.greenlet = greenlet.greenlet(runFuture)
future = future._switch(future)
if scoop.DEBUG:
lastDebugTs = time.time()
while not scoop.IS_ORIGIN or future.parentId != rootId or not future._ended(
):
if scoop.DEBUG and time.time(
) - lastDebugTs < scoop.TIME_BETWEEN_PARTIALDEBUG:
from scoop import _debug
_debug.writeWorkerDebug(
debug_stats, QueueLength,
"debug/partial-{0}".format(round(time.time(), -1)))
# process future
if future._ended():
# future is finished
if future.id.worker != scoop.worker:
# future is not local
execQueue.sendResult(future)
future = execQueue.pop()
else:
# future is local, parent is waiting
if future.index is not None:
parent = futureDict[future.parentId]
if parent.exceptionValue is None:
future = parent._switch(future)
else:
future = execQueue.pop()
else:
future = execQueue.pop()
else:
# future is in progress; run next future from pending execution queue.
future = execQueue.pop()
if not future._ended() and future.greenlet is None:
# initialize if the future hasn't started
future.greenlet = greenlet.greenlet(runFuture)
future = future._switch(future)
execQueue.shutdown()
if future.exceptionValue:
raise future.exceptionValue
return future.resultValue
def runController(callable_, *args, **kargs):
"""Callable greenlet implementing controller logic."""
global execQueue
# initialize and run root future
rootId = (-1, 0)
# initialise queue
if execQueue is None:
execQueue = FutureQueue()
sys.excepthook = advertiseBrokerWorkerDown
if scoop.DEBUG:
from scoop import _debug
_debug.redirectSTDOUTtoDebugFile()
# TODO: Make that a function
# Wait until we received the main module if we are a headless slave
headless = scoop.CONFIGURATION.get("headless", False)
if not scoop.MAIN_MODULE:
# If we're not the origin and still don't have our main_module,
# wait for it and then import it as module __main___
main = scoop.shared.getConst('__MAIN_MODULE__', timeout=float('inf'))
directory_name = tempfile.mkdtemp()
os.chdir(directory_name)
scoop.MAIN_MODULE = main.writeFile(directory_name)
from .bootstrap.__main__ import Bootstrap as SCOOPBootstrap
newModule = SCOOPBootstrap.setupEnvironment()
sys.modules['__main__'] = newModule
elif scoop.IS_ORIGIN and headless and scoop.MAIN_MODULE:
# We're the origin, share our main_module
scoop.shared.setConst(
__MAIN_MODULE__=scoop.encapsulation.ExternalEncapsulation(
scoop.MAIN_MODULE,
)
)
# TODO: use modulefinder to share every local dependency of
# main module
# launch future if origin or try to pickup a future if slave worker
if scoop.IS_ORIGIN:
future = Future(rootId, callable_, *args, **kargs)
else:
future = execQueue.pop()
future.greenlet = greenlet.greenlet(runFuture)
future = future._switch(future)
if scoop.DEBUG:
lastDebugTs = time.time()
while not scoop.IS_ORIGIN or future.parentId != rootId or not future._ended():
if scoop.DEBUG and time.time() - lastDebugTs > scoop.TIME_BETWEEN_PARTIALDEBUG:
_debug.writeWorkerDebug(
debug_stats,
QueueLength,
"debug/partial-{0}".format(
round(time.time(), -1)
)
)
lastDebugTs = time.time()
# process future
if future._ended():
# future is finished
if future.id[0] != scoop.worker:
# future is not local
execQueue.sendResult(future)
future = execQueue.pop()
else:
# future is local, parent is waiting
if future.index is not None:
try:
parent = futureDict[future.parentId]
except KeyError:
# Job has no parent here (probably children restart)
future = execQueue.pop()
else:
if parent.exceptionValue is None:
future = parent._switch(future)
else:
future = execQueue.pop()
else:
future = execQueue.pop()
else:
# future is in progress; run next future from pending execution queue.
future = execQueue.pop()
if not future._ended() and future.greenlet is None:
# initialize if the future hasn't started
future.greenlet = greenlet.greenlet(runFuture)
future = future._switch(future)
execQueue.shutdown()
if future.exceptionValue:
print(future.exceptionTraceback)
sys.exit(1)
return future.resultValue
def runController(callable_, *args, **kargs):
"""Callable greenlet implementing controller logic."""
global execQueue
# initialize and run root future
rootId = (-1, 0)
# initialise queue
if execQueue is None:
execQueue = FutureQueue()
sys.excepthook = advertiseBrokerWorkerDown
if scoop.DEBUG:
from scoop import _debug
_debug.redirectSTDOUTtoDebugFile()
# TODO: Make that a function
# Wait until we received the main module if we are a headless slave
headless = scoop.CONFIGURATION.get("headless", False)
if not scoop.MAIN_MODULE:
# If we're not the origin and still don't have our main_module,
# wait for it and then import it as module __main___
main = scoop.shared.getConst('__MAIN_MODULE__', timeout=float('inf'))
directory_name = tempfile.mkdtemp()
os.chdir(directory_name)
scoop.MAIN_MODULE = main.writeFile(directory_name)
from .bootstrap.__main__ import Bootstrap as SCOOPBootstrap
newModule = SCOOPBootstrap.setupEnvironment()
sys.modules['__main__'] = newModule
elif scoop.IS_ORIGIN and headless and scoop.MAIN_MODULE:
# We're the origin, share our main_module
scoop.shared.setConst(
__MAIN_MODULE__=scoop.encapsulation.ExternalEncapsulation(
scoop.MAIN_MODULE,
)
)
# TODO: use modulefinder to share every local dependency of
# main module
# launch future if origin or try to pickup a future if slave worker
if scoop.IS_ORIGIN:
future = Future(rootId, callable_, *args, **kargs)
else:
future = execQueue.pop()
future.greenlet = greenlet.greenlet(runFuture)
future = future._switch(future)
if scoop.DEBUG:
lastDebugTs = time.time()
while not scoop.IS_ORIGIN or future.parentId != rootId or not future._ended():
if scoop.DEBUG and time.time() - lastDebugTs > scoop.TIME_BETWEEN_PARTIALDEBUG:
_debug.writeWorkerDebug(
debug_stats,
QueueLength,
"debug/partial-{0}".format(
round(time.time(), -1)
)
)
lastDebugTs = time.time()
# At this point , the future is either completed or in progress
# in progress => its runFuture greenlet is in progress
# process future and get new future / result future
waiting_parent = False
if future.isDone:
if future.isReady:
# future is completely processed and ready to be returned This return is
# executed only if the parent is waiting for this particular future.
# Note: orphaned futures are caused by... what again?
execQueue.sendReadyStatus(future)
if future.index is not None:
# This means that this particular future is being waited upon by a
# parent (see futures._waitAny())
if future.parentId in futureDict:
parent = futureDict[future.parentId]
if parent.exceptionValue is None:
waiting_parent = True
else:
execQueue.finalizeFuture(future)
if future.isReady and waiting_parent:
# This means that this future must be returned to the parent future greenlet
# by this, one of 2 things happen, the parent completes execution and
# returns itself along with its results and isDone set to True, or, in case
# it has spawned a sub-future or needs to wait on another future, it returns
# itself in an inprogress state (isDone = False)
future = parent._switch(future)
else:
# This means that the future is either ready and the parent is not waiting,
# or the future is in progress. This happens when the future is a parent
# future whose greenlet has returned itself in an incomplete manner. check
# the above comment
#
# In both the above cases, the future can safely be dropped/ignored and the
# next future picked up from the queue for processing.
future = execQueue.pop()
if not future._ended() and future.greenlet is None:
# This checks for the case of a not-yet-started-execution future that is
# returned from the queue, (This can only happen if execQueue.pop is called)
# and starts the execution
future.greenlet = greenlet.greenlet(runFuture)
future = future._switch(future)
# Special case of removing the root future from the futureDict
scoop._control.delFuture(future)
execQueue.shutdown()
if future.exceptionValue:
print(future.exceptionTraceback)
sys.exit(1)
return future.resultValue
