def _startup(rootFuture, *args, **kargs): """Initializes the SCOOP environment. :param rootFuture: Any callable object (function or class object with *__call__* method); this object will be called once and allows the use of parallel calls inside this object. :param args: A tuple of positional arguments that will be passed to the callable object. :param kargs: A dictionary of additional keyword arguments that will be passed to the callable object. :returns: The result of the root Future. Be sure to launch your root Future using this method.""" import greenlet global _controller _controller = greenlet.greenlet(control.runController) try: result = _controller.switch(rootFuture, *args, **kargs) except scoop._comm.Shutdown: result = None control.execQueue.shutdown() if scoop.DEBUG: from scoop import _debug _debug.writeWorkerDebug(control.debug_stats, control.QueueLength) return result
def shutdown(self): """Shutdown the ressources used by the queue""" self.socket.shutdown() if scoop: if scoop.DEBUG: from scoop import _debug _debug.writeWorkerDebug( scoop._control.debug_stats, scoop._control.QueueLength, )
def runController(callable_, *args, **kargs): """Callable greenlet implementing controller logic.""" global execQueue # initialize and run root future rootId = FutureId(-1, 0) # initialise queue if execQueue is None: execQueue = FutureQueue() sys.excepthook = advertiseBrokerWorkerDown # TODO: Make that a function # Wait until we received the main module if we are a headless slave headless = scoop.CONFIGURATION.get("headless", False) if not scoop.MAIN_MODULE: # If we're not the origin and still don't have our main_module, # wait for it and then import it as module __main___ main = scoop.shared.getConst('__MAIN_MODULE__', timeout=float('inf')) directory_name = tempfile.mkdtemp() os.chdir(directory_name) scoop.MAIN_MODULE = main.writeFile(directory_name) from .bootstrap.__main__ import Bootstrap as SCOOPBootstrap newModule = SCOOPBootstrap.setupEnvironment() sys.modules['__main__'] = newModule elif scoop.IS_ORIGIN and headless and scoop.MAIN_MODULE: # We're the origin, share our main_module scoop.shared.setConst( __MAIN_MODULE__=scoop.encapsulation.ExternalEncapsulation( scoop.MAIN_MODULE, )) # TODO: use modulefinder to share every local dependency of # main module # launch future if origin or try to pickup a future if slave worker if scoop.IS_ORIGIN: future = Future(rootId, callable_, *args, **kargs) else: future = execQueue.pop() future.greenlet = greenlet.greenlet(runFuture) future = future._switch(future) if scoop.DEBUG: lastDebugTs = time.time() while not scoop.IS_ORIGIN or future.parentId != rootId or not future._ended( ): if scoop.DEBUG and time.time( ) - lastDebugTs < scoop.TIME_BETWEEN_PARTIALDEBUG: from scoop import _debug _debug.writeWorkerDebug( debug_stats, QueueLength, "debug/partial-{0}".format(round(time.time(), -1))) # process future if future._ended(): # future is finished if future.id.worker != scoop.worker: # future is not local execQueue.sendResult(future) future = execQueue.pop() else: # future is local, parent is waiting if future.index is not None: parent = futureDict[future.parentId] if parent.exceptionValue is None: future = parent._switch(future) else: future = execQueue.pop() else: future = execQueue.pop() else: # future is in progress; run next future from pending execution queue. future = execQueue.pop() if not future._ended() and future.greenlet is None: # initialize if the future hasn't started future.greenlet = greenlet.greenlet(runFuture) future = future._switch(future) execQueue.shutdown() if future.exceptionValue: raise future.exceptionValue return future.resultValue
def runController(callable_, *args, **kargs): """Callable greenlet implementing controller logic.""" global execQueue # initialize and run root future rootId = (-1, 0) # initialise queue if execQueue is None: execQueue = FutureQueue() sys.excepthook = advertiseBrokerWorkerDown if scoop.DEBUG: from scoop import _debug _debug.redirectSTDOUTtoDebugFile() # TODO: Make that a function # Wait until we received the main module if we are a headless slave headless = scoop.CONFIGURATION.get("headless", False) if not scoop.MAIN_MODULE: # If we're not the origin and still don't have our main_module, # wait for it and then import it as module __main___ main = scoop.shared.getConst('__MAIN_MODULE__', timeout=float('inf')) directory_name = tempfile.mkdtemp() os.chdir(directory_name) scoop.MAIN_MODULE = main.writeFile(directory_name) from .bootstrap.__main__ import Bootstrap as SCOOPBootstrap newModule = SCOOPBootstrap.setupEnvironment() sys.modules['__main__'] = newModule elif scoop.IS_ORIGIN and headless and scoop.MAIN_MODULE: # We're the origin, share our main_module scoop.shared.setConst( __MAIN_MODULE__=scoop.encapsulation.ExternalEncapsulation( scoop.MAIN_MODULE, ) ) # TODO: use modulefinder to share every local dependency of # main module # launch future if origin or try to pickup a future if slave worker if scoop.IS_ORIGIN: future = Future(rootId, callable_, *args, **kargs) else: future = execQueue.pop() future.greenlet = greenlet.greenlet(runFuture) future = future._switch(future) if scoop.DEBUG: lastDebugTs = time.time() while not scoop.IS_ORIGIN or future.parentId != rootId or not future._ended(): if scoop.DEBUG and time.time() - lastDebugTs > scoop.TIME_BETWEEN_PARTIALDEBUG: _debug.writeWorkerDebug( debug_stats, QueueLength, "debug/partial-{0}".format( round(time.time(), -1) ) ) lastDebugTs = time.time() # process future if future._ended(): # future is finished if future.id[0] != scoop.worker: # future is not local execQueue.sendResult(future) future = execQueue.pop() else: # future is local, parent is waiting if future.index is not None: try: parent = futureDict[future.parentId] except KeyError: # Job has no parent here (probably children restart) future = execQueue.pop() else: if parent.exceptionValue is None: future = parent._switch(future) else: future = execQueue.pop() else: future = execQueue.pop() else: # future is in progress; run next future from pending execution queue. future = execQueue.pop() if not future._ended() and future.greenlet is None: # initialize if the future hasn't started future.greenlet = greenlet.greenlet(runFuture) future = future._switch(future) execQueue.shutdown() if future.exceptionValue: print(future.exceptionTraceback) sys.exit(1) return future.resultValue
def runController(callable_, *args, **kargs): """Callable greenlet implementing controller logic.""" global execQueue # initialize and run root future rootId = (-1, 0) # initialise queue if execQueue is None: execQueue = FutureQueue() sys.excepthook = advertiseBrokerWorkerDown if scoop.DEBUG: from scoop import _debug _debug.redirectSTDOUTtoDebugFile() # TODO: Make that a function # Wait until we received the main module if we are a headless slave headless = scoop.CONFIGURATION.get("headless", False) if not scoop.MAIN_MODULE: # If we're not the origin and still don't have our main_module, # wait for it and then import it as module __main___ main = scoop.shared.getConst('__MAIN_MODULE__', timeout=float('inf')) directory_name = tempfile.mkdtemp() os.chdir(directory_name) scoop.MAIN_MODULE = main.writeFile(directory_name) from .bootstrap.__main__ import Bootstrap as SCOOPBootstrap newModule = SCOOPBootstrap.setupEnvironment() sys.modules['__main__'] = newModule elif scoop.IS_ORIGIN and headless and scoop.MAIN_MODULE: # We're the origin, share our main_module scoop.shared.setConst( __MAIN_MODULE__=scoop.encapsulation.ExternalEncapsulation( scoop.MAIN_MODULE, ) ) # TODO: use modulefinder to share every local dependency of # main module # launch future if origin or try to pickup a future if slave worker if scoop.IS_ORIGIN: future = Future(rootId, callable_, *args, **kargs) else: future = execQueue.pop() future.greenlet = greenlet.greenlet(runFuture) future = future._switch(future) if scoop.DEBUG: lastDebugTs = time.time() while not scoop.IS_ORIGIN or future.parentId != rootId or not future._ended(): if scoop.DEBUG and time.time() - lastDebugTs > scoop.TIME_BETWEEN_PARTIALDEBUG: _debug.writeWorkerDebug( debug_stats, QueueLength, "debug/partial-{0}".format( round(time.time(), -1) ) ) lastDebugTs = time.time() # At this point , the future is either completed or in progress # in progress => its runFuture greenlet is in progress # process future and get new future / result future waiting_parent = False if future.isDone: if future.isReady: # future is completely processed and ready to be returned This return is # executed only if the parent is waiting for this particular future. # Note: orphaned futures are caused by... what again? execQueue.sendReadyStatus(future) if future.index is not None: # This means that this particular future is being waited upon by a # parent (see futures._waitAny()) if future.parentId in futureDict: parent = futureDict[future.parentId] if parent.exceptionValue is None: waiting_parent = True else: execQueue.finalizeFuture(future) if future.isReady and waiting_parent: # This means that this future must be returned to the parent future greenlet # by this, one of 2 things happen, the parent completes execution and # returns itself along with its results and isDone set to True, or, in case # it has spawned a sub-future or needs to wait on another future, it returns # itself in an inprogress state (isDone = False) future = parent._switch(future) else: # This means that the future is either ready and the parent is not waiting, # or the future is in progress. This happens when the future is a parent # future whose greenlet has returned itself in an incomplete manner. check # the above comment # # In both the above cases, the future can safely be dropped/ignored and the # next future picked up from the queue for processing. future = execQueue.pop() if not future._ended() and future.greenlet is None: # This checks for the case of a not-yet-started-execution future that is # returned from the queue, (This can only happen if execQueue.pop is called) # and starts the execution future.greenlet = greenlet.greenlet(runFuture) future = future._switch(future) # Special case of removing the root future from the futureDict scoop._control.delFuture(future) execQueue.shutdown() if future.exceptionValue: print(future.exceptionTraceback) sys.exit(1) return future.resultValue