def handle_service_result(response: Dict, outputs, name: str) -> Dict:
d = {}
for k, v in response.items():
log.debug(" handling req result props: {0} -> {1}".format(k, str(v)))
if k in outputs:
d[name + "." + k] = v
return d
def handle_python(self, _task: OrcaTask):
log.debug(" exec python file : " + _task.python)
resolved_file = self._resolve_file_path(_task.python, ".py")
config = _task.config
try:
if resolved_file is None:
exec(_task.python, _task.locals)
else:
with open(resolved_file, 'r') as script:
_python = script.read()
exec(_python, _task.locals)
if 'callable' in config:
call_str = self.__get_call_string(config, _python)
exec(call_str, _task.locals)
_task.status = "success"
except IndexError:
raise ExecutionError(
'The function {0} was not defined in the file: {1}'.format(_task.config.get('callable'), _task.python)
)
except BaseException as e:
_task.status = "failed"
log.debug(str(e))
raise
# remove after execution
keys_to_remove = [k for k in _task.locals if
k not in _task.outputs and k not in _task.inputs]
for key in keys_to_remove:
del _task.locals[key]
def _resolve_file_path(self, name: str, ext: str) -> str:
""" resolve the full qualified path name"""
def resolve_file_path(handler: OrcaHandler, _name: str) -> str:
""" resolve the full qualified path name"""
if os.path.isfile(_name):
return _name
# otherwise find the relative dir
elif hasattr(handler, 'config'):
yaml_dir = handler.config.get_yaml_dir()
else:
yaml_dir = "."
rel_path = os.path.join(yaml_dir, _name)
if os.path.isfile(rel_path):
# path relative to yaml file
return rel_path
else:
# this should be a file but it's not.
raise ConfigurationError(
'File not found: "{0}"'.format(_name))
# check to see if the value ends with an extension
if name.endswith(ext):
# check if its an absolute path
return resolve_file_path(self, name)
# check if its a variable
elif name.startswith('var.'):
try:
name = eval(str(name), globals())
except Exception as e:
# ok, never mind
log.debug(e)
if not name:
return name
return resolve_file_path(self, name)
def _handle_task(self, task_dict: Dict) -> OrcaTask:
name = task_dict.get('task', None)
# check the symbol table for task name to be an unique and valid name
self._check_symtable(name, task_dict)
# task_locals are the resolved inputs, they will be used for
# execution
task_locals = self.resolve_task_inputs(task_dict)
_task = OrcaTask(task_dict, task_locals)
log.debug("task '{0}' locals pre: {1}".format(_task.name, _task.locals))
# select the handler and call handle
handle = self.select_handler(task_dict)
log.info('Starting task {0}'.format(name))
handle(_task)
log.info('Task {0} completed'.format(name, ))
log.debug("task '{0}' locals post: {1}".format(
_task.name, _task.locals))
# put the task_locals into the global task dictonary
# this includes input and outputs
task[_task.name] = {}
for k, v in _task.locals.items():
task[_task.name][k] = v
return _task
def _resolve_file_path(self, name: str, ext: str) -> str:
""" resolve the full qualified path name"""
# potential value resolution if this should be a file name
try:
name = eval(str(name), globals())
except Exception as e:
# ok, never mind
log.debug(e)
if os.path.isfile(name):
return name
else:
# maybe change this, because of testing
if hasattr(self, 'config'):
yaml_dir = self.config.get_yaml_dir()
else:
yaml_dir = "."
rel_path = os.path.join(yaml_dir, name)
if os.path.isfile(rel_path):
# path relative to yaml file
return rel_path
else:
if name.endswith(ext):
# this should be a file but it's not.
raise ConfigurationError(
'File not found: "{0}"'.format(name))
return None
def _create_entry(self, task: OrcaTask) -> Dict:
"""Create a dictionary entry to record in a ledger"""
d = {
# the workflow file
'orca_file': os.path.abspath(self.config.get_yaml_file()),
# the 'version' entry in the workflow file
'orca_id': self.config.get_version(),
# the 'version' entry in the workflow file (e.g. gitattribute)
'orca_name': self.config.get_name(),
'task_name': task.name, # the task name
'task_uuid': str(self._id), # the uuid of the current run
'task_status': task.status, # status of the run
'task_time': str(datetime.now()), # task execution time
'task_data': task.locals, # task data
}
if log.isEnabledFor(logging.DEBUG):
log.debug('ledger: {0}'.format(d))
return d
def _handle_sequence(self, sequence: Dict) -> None:
for step in sequence:
node = next(iter(step))
if node == "task":
log.debug(" ---- task: '{}'".format(step['task']))
self._handle_task(step)
elif node == "if":
log.debug(" ---- if: '{}'".format(step['if']))
self._handle_if(step)
elif node == "for":
log.debug(" ---- for: '{}'".format(step['for']))
self._handle_for(step)
elif node.startswith("fork"):
log.debug(" ---- fork: ")
self._handle_fork(step['fork'])
elif node == "switch":
log.debug(" ---- switch: '{}'".format(step['switch']))
self._handle_switch(step)
else:
raise ConfigurationError(
'Invalid step in job: "{0}"'.format(node))
def close(self) -> None:
self.mongo.close()
log.debug('closed: {0}'.format(self.mongo))
def __init__(self, con: List[str]):
super().__init__()
self.mongo = pymongo.MongoClient('mongodb://{0}/'.format(con[0]))
self.col = self.mongo[con[1]][con[2]]
log.debug('Mongo Ledger: {0} for {1}'.format(self.mongo,
"/".join(con)))
def close(self) -> None:
self.f.close()
log.debug('closed: {0}'.format(self.f))
def __init__(self, file: str):
super().__init__()
self.f = open(file, 'a+')
log.debug('JSON Ledger: {0}'.format(self.f))
def close(self) -> None:
self.kafka.close()
log.debug('closed: {0}'.format(self.kafka))
