def start(self):
"""
Method to start the job associated to this Runner
@ In, None
@ Out, None
"""
try:
if im.isLibAvail("ray"):
self.thread = self.functionToRun(*self.args)
else:
self.thread = self.__ppserver.submit(
self.functionToRun,
args=self.args,
depfuncs=(),
modules=tuple([self.functionToRun.__module__] + list(
set(
utils.returnImportModuleString(
inspect.getmodule(self.functionToRun), True)))
))
self.trackTime('runner_started')
self.started = True
gc.collect()
return
except Exception as ae:
#Uncomment if you need the traceback
self.exceptionTrace = sys.exc_info()
#exc_type, exc_value, exc_traceback = sys.exc_info()
#import traceback
#traceback.print_exception(exc_type, exc_value, exc_traceback)
self.raiseAWarning(
self.__class__.__name__ + " job " + self.identifier +
" failed with error:" + str(ae) + " !", 'ExceptedError')
self.returnCode = -1
def _collectRunnerResponse(self):
"""
Method to add the process response in the internal variable (pointer)
self.__runReturn
@ In, None
@ Out, None
"""
if not self.hasBeenAdded:
if self.thread is not None:
self.runReturn = ray.get(
self.thread) if im.isLibAvail("ray") else self.thread()
else:
self.runReturn = None
self.hasBeenAdded = True
def isDone(self):
"""
Method to check if the calculation associated with this Runner is finished
@ In, None
@ Out, finished, bool, is it finished?
"""
## If the process has not been started yet, then return False
if not self.started:
return False
if self.thread is None:
return True
else:
return (self.thread in ray.wait([self.thread], timeout=waitTimeOut)
[0]) if im.isLibAvail("ray") else self.thread.finished
def __init__(self,
messageHandler,
args,
functionToRun,
identifier=None,
metadata=None,
uniqueHandler="any",
profile=False):
"""
Init method
@ In, messageHandler, MessageHandler object, the global RAVEN message
handler object
@ In, args, list, this is a list of arguments that will be passed as
function parameters into whatever method is stored in functionToRun.
e.g., functionToRun(*args)
@ In, functionToRun, method or function, function that needs to be run
@ In, identifier, string, optional, id of this job
@ In, metadata, dict, optional, dictionary of metadata associated with
this run
@ In, forceUseThreads, bool, optional, flag that, if True, is going to
force the usage of multi-threading even if parallel python is activated
@ In, uniqueHandler, string, optional, it is a special keyword attached to
this runner. For example, if present, to retrieve this runner using the
method jobHandler.getFinished, the uniqueHandler needs to be provided.
If uniqueHandler == 'any', every "client" can get this runner
@ In, profile, bool, optional, if True then timing statements are printed
during deconstruction.
@ Out, None
"""
## First, allow the base class to handle the commonalities
## We keep the command here, in order to have the hook for running exec
## code into internal models
if not im.isLibAvail("ray"):
self.__ppserver, args = args[0], args[1:]
super(DistributedMemoryRunner,
self).__init__(messageHandler, args, functionToRun, identifier,
metadata, uniqueHandler, profile)
# limitations under the License.
"""
Created on March 29, 2020
@author: alfoa
"""
#Internal Modules---------------------------------------------------------------
from utils import importerUtils as im
from utils.utils import Object
#Internal Modules End-----------------------------------------------------------
#External Modules---------------------------------------------------------------
# for internal parallel
## TODO: REMOVE WHEN RAY AVAILABLE FOR WINDOWOS
_remote = None
if im.isLibAvail("ray"):
from ray import remote as _remote
# end internal parallel module
#External Modules End-----------------------------------------------------------
class Parallel(object):
"""
RAVEN parallel decorator
It is used to mask ray parallel remote
decorator and to allow for direct call of
the underlying function (via _function attribute)
i.e. :
- if ray is available and needs to be used,
the decorated function (or class) will need to be called as following:
functionName.remote(*args, **kwargs)
import threading
from random import randint
import socket
import time
#External Modules End-----------------------------------------------------------
#Internal Modules---------------------------------------------------------------
from utils import importerUtils as im
from utils import utils
from BaseClasses import BaseType
import MessageHandler
import Runners
import Models
# for internal parallel
## TODO: REMOVE WHEN RAY AVAILABLE FOR WINDOWS
_rayAvail = im.isLibAvail("ray")
if _rayAvail:
import ray
else:
import pp
# end internal parallel module
#Internal Modules End-----------------------------------------------------------
## FIXME: Finished jobs can bog down the queue waiting for other objects to take
## them away. Can we shove them onto a different list and free up the job queue?
class JobHandler(MessageHandler.MessageUser):
"""
JobHandler class. This handles the execution of any job in the RAVEN
framework