def execute(self, taskSpec: Sequence[str]) -> Response:
clientId = self.elem(taskSpec, 0)
runargs = self.getRunArgs(taskSpec)
jobId = runargs.get("jobId", self.elem(taskSpec, 2, Job.randomStr(8)))
proj = runargs.get("proj", "proj-" + Job.randomStr(4))
exp = runargs.get("exp", "exp-" + Job.randomStr(4))
process_name = self.elem(taskSpec, 2)
runargs["ncores"] = self.processManager.ncores.items()[0][1]
dataInputsSpec = self.elem(taskSpec, 3)
self.setExeStatus(clientId, jobId,
"executing " + process_name + "-> " + dataInputsSpec)
self.logger.info(" @@E: Executing " + process_name + "-> " +
dataInputsSpec + ", jobId = " + jobId +
", runargs = " + str(runargs))
try:
job = Job.new(jobId, proj, exp, process_name, dataInputsSpec,
runargs, 1.0)
execHandler: ExecHandler = self.addHandler(
clientId, jobId,
ExecHandler(clientId, job, self, workers=job.workers))
execHandler.start()
return Message(clientId, jobId, execHandler.filePath)
except Exception as err:
self.logger.error("Caught execution error: " + str(err))
traceback.print_exc()
return Message(clientId, jobId, str(err))
def request(self, requestSpec: Dict, inputs: List[TaskResult] = None, **kwargs ) -> TaskHandle:
rid: str = kwargs.get('rid', Job.randomStr(4) )
cid: str = kwargs.get('cid', Job.randomStr(4) )
self.logger.info( f"EDAS Endpoint--> processing rid {rid}")
proj = requestSpec.get("proj", "proj-" + Job.randomStr(4) )
exp = requestSpec.get("exp", "exp-" + Job.randomStr(4) )
try:
job = Job.create( rid, proj, exp, 'exe', requestSpec, inputs, {}, 1.0 )
execHandler: TaskExecHandler = self.addHandler(rid, TaskExecHandler(cid, job))
execHandler.execJob( job )
return execHandler
except Exception as err:
self.logger.error( "Caught execution error: " + str(err) )
traceback.print_exc()
return TaskHandle(rid=rid, cid=cid, status = Status.ERROR, error = TaskExecHandler.getErrorReport(err))
def getExpFile(self,
proj: str,
exp: str,
name: str,
type: str = "nc") -> str:
expDir = self.getExpDir(proj, exp)
return expDir + "/" + name + "-" + Job.randomStr(6) + "." + type
def __init__(self, clientId: str, jobId: str, **kwargs):
self.logger = EDASLogger.getLogger()
self.clientId = clientId
self.jobId = jobId
self.cacheDir = kwargs.get("cache", "/tmp")
self.workers = kwargs.get("workers", 1)
self.start_time = time.time()
self.filePath = self.cacheDir + "/" + Job.randomStr(6) + ".nc"
def __init__(self,
name: str = None,
dsets: "OrderedDict[str, EDASDataset]" = None):
from edas.process.task import Job
self._name = Job.randomStr(6) if name is None else name
self._datasets: OrderedDict[
str, EDASDataset] = OrderedDict() if dsets is None else dsets
print(" $$$$ DsetCol(" + self._name + ").init: " + self.arrayIds)
def __init__(self, name: str, _domain: str, metadata: Dict[str, Any]):
from edas.process.task import Job
super(WorkflowNode, self).__init__(name, metadata)
self.domain: str = _domain
nameToks = name.split(":") if ":" in name else name.split(".")
self.module: str = nameToks[0]
assert len(
nameToks
) > 1, "Expected '<module>.<operation>' format, received: '{}'".format(
name)
self.op: str = nameToks[1]
self.axes: List[str] = self._getAxes("axis") + self._getAxes("axes")
self._connectors: List[OperationConnector] = []
self._masterNode: MasterNodeWrapper = None
self._addWorkflowConnectors()
self._instanceId = self.name + "-" + Job.randomStr(6)