def test_check_err_create_bad_dom(self):
"""
Tests that CheckEngine produces expected CheckStatus, when checking domain error exists
with the CustomizedAlgo creation Rest service.
"""
# Re-activate checking rules
CheckEngine.set_checking_rules(
FactoryCatalogue.TYPE_CHECKING_FUNCTIONS)
my_custom_outside_domain = self.init_rsrc_outside_domain()
http_req = self.request_factory.post(
path=reverse('pattern_custom_algos_dummy'),
data=CustomizedAlgoWs(my_custom_outside_domain).to_json(),
content_type="application/json_util")
http_response = create_custom_algo(http_req)
# Display the check result
result = json_io.decode_json_from_http_response(http_response)
self.info("response loaded={}".format(result))
self.info("response status={}".format(http_response.status_code))
self.assertTrue(http_response.status_code ==
HttpResponseFactory.BAD_REQUEST_HTTP_STATUS)
def init_tu(cls):
"""
To be called in the setUpClass of each test:
- init the logger
- empties the checking rules: CheckEngine.set_checking_rules({})
:param cls:
:type cls:
"""
cls.cat_initialized = False
cls.init_logger()
# for each test: the checking rules are emptied: default behaviour
# which may be modified in each setUpClass method
CheckEngine.set_checking_rules({})
def __init__(self, executable_algo, debug=False, dao_managed=False):
super(PythonLocalExecEngine, self).__init__(executable_algo, debug,
dao_managed)
# internal use: string configuring the python function
self.__lib_path = None
# internal use: evaluated python function definition
self.__evaluated_python_function = None
self.__checker = CheckEngine(
checked_resource=self.executable_algo.custom_algo,
checked_value_context="Running custom algo from " +
self.__class__.__name__)
def check_edited_params(cls, business_algo, checked_value_context):
"""
Apply the consistency checks on the business_algo
:param cls:
:type cls:
:param business_algo: the customized algo
:type business_algo: CustomizedAlgo
:param checked_value_context: additional information about the processing context of the check:
passed to the CheckEngine constructor.
:return: the checkStatus completed with these checks
:rtype: checkStatus
"""
engine = CheckEngine(business_algo, checked_value_context)
return engine.check_customized_values()
def commons_tear_down_class(cls):
super(CommonsCatalogueTests, cls).commons_tear_down_class()
CheckEngine.set_checking_rules({})
class PythonLocalExecEngine(ExecEngine):
"""
This Engine allows the execution of python function as defined by self.executable_algo:
see run_command() documentation.
Required on executable_algo.custom_algo.implementation (typed Implementation):
- implementation._execution_plugin is "apps.algo.execute.models.business.exec_engine.PythonLocalExecEngine"
(this is required by FacadeExecution)
- implementation.lib_path must respect syntax: <function module>::<function name>
Usage:
ea = FacadeExecution.factory.build...Algo(...)
ea.set_data_source(name, source)
...
ea.set_data_receiver( "result", source )
...
engine = PythonLocalExecEngine( ea )
# ExecStatus instance is gathering global information about execution
exec_status = engine.execute()
print( exec_status )
# Results: available on data receivers defined on ExecutableAlgorithm ea:
they are not stocked by the engine
result = ea.get_data_receiver("result").get_value()
See also:
FacadeExecution which provides easy ways to launch algo executions
"""
def __init__(self, executable_algo, debug=False, dao_managed=False):
super(PythonLocalExecEngine, self).__init__(executable_algo, debug,
dao_managed)
# internal use: string configuring the python function
self.__lib_path = None
# internal use: evaluated python function definition
self.__evaluated_python_function = None
self.__checker = CheckEngine(
checked_resource=self.executable_algo.custom_algo,
checked_value_context="Running custom algo from " +
self.__class__.__name__)
def __evaluate_python_function(self):
"""
Private method called once on the engine instance: evaluate the python function
and assign it to the self.__evaluated_python_function
"""
try:
my_custo_algo = self.executable_algo.custom_algo
my_implem = my_custo_algo.implementation
self.__lib_path = my_implem.library_address
self.add_msg("lib_path=%s" % self.__lib_path)
my_parsed_lib_path = self.__parse_lib_path(self.__lib_path)
# already checked: parsed parts
my_module_path = my_parsed_lib_path[0]
self.add_msg("module path=%s" % my_module_path)
my_function = my_parsed_lib_path[1]
self.add_msg("function name=%s" % my_function)
# !!! keep this variable my_module declared !!!
my_module = importlib.import_module(my_module_path)
evaluated_function_str = "my_module.%s" % my_function
self.add_msg('evaluating "%s"' % evaluated_function_str)
self.__evaluated_python_function = eval(evaluated_function_str)
except Exception as err:
trace_back = sys.exc_info()[2]
LOGGER.exception(err)
raise EngineException(
"PythonLocalExecEngine failed to evaluate configured python function [%s] from executable algo [%s]"
% (self.__lib_path,
self.executable_algo)).with_traceback(trace_back)
@staticmethod
def __parse_lib_path(lib_path):
split_after_module = lib_path.split("::")
return split_after_module
def run_command(self):
"""
Implements the calling of python function defined by self.executable_algo:
- python function is evaluated once: it is fully defined by executable_algo.custom_algo.implementation:
- implementation.lib_path must respect syntax: <function module>::<function name>
- then evaluated function is called according to input and output profiles configured in
executable_algo.custom_algo.implementation:
- input arguments are consumed on self.executable_algo: using ExecutableAlgo::consume(...)
and ExecutableAlgo::get_ordered_input_names(...)
- output result(s) are produced on self.executable_algo: using ExecutableAlgo::produce(...)
and ExecutableAlgo::get_ordered_output_names(...)
"""
if self.__evaluated_python_function is None:
self.__evaluate_python_function()
args = []
try:
LOGGER.debug("self.executable_algo.get_ordered_input_names() : %s",
str(self.executable_algo.get_ordered_input_names()))
implem = self.executable_algo.custom_algo.implementation
for input_name in self.executable_algo.get_ordered_input_names():
profile_item = implem.find_by_name_input_item(input_name)
value_consumed = self.executable_algo.consume_value(input_name)
self.__checker.check_type(profile_item, value_consumed)
status = self.__checker.check_domain(profile_item,
value_consumed)
if status.has_errors():
raise CheckError("CheckEngine has detected errors.",
status)
args.append(value_consumed)
except CheckError as error:
trace_back = sys.exc_info()[2]
msg = "PythonLocalExecEngine aborted run: incorrect inputs " + \
"from executed python function {} for algo {}: see detailed status={}"
readable_status = json.dumps(obj=error.status().to_dict(),
indent=2)
# We consider that the bad input error is almost like AlgoException:
# because the EngineException errors are reserved to technical errors on the server
#
raise AlgoException(
msg.format(self.__lib_path, self.executable_algo,
readable_status), error).with_traceback(trace_back)
except Exception as err_pre:
trace_back = sys.exc_info()[2]
raise EngineException(
"PythonLocalExecEngine failed to consume inputs from executable algo [%s]"
% self.executable_algo, err_pre).with_traceback(trace_back)
result = tuple()
try:
result = self.__evaluated_python_function(*args)
except Exception as err:
trace_back = sys.exc_info()[2]
raise AlgoException(
"PythonLocalExecEngine received error from executed python function [%s] for algo [%s]"
% (self.__lib_path, self.executable_algo),
err).with_traceback(trace_back)
try:
res_names = self.executable_algo.get_ordered_output_names()
if type(result) is tuple:
assert (len(result) == len(res_names))
for output_name, output_value in zip(res_names, result):
self.executable_algo.produce_value(output_name,
output_value)
elif len(res_names) == 1:
LOGGER.info("Expected unique result, with name=" +
res_names[0])
if result is None:
LOGGER.warning(
"No result => skipped step: executable_algo.produce_values : there is no output defined "
"for implementation")
LOGGER.warning(
str(self.executable_algo.get_custom_algo().
get_implementation()))
else:
self.executable_algo.produce_value(res_names[0], result)
else:
# void functions : like statistics builders ...
LOGGER.info(
"No result => skipped step: executable_algo.produce_values : there is no output defined for "
"implementation")
except Exception as err_post:
trace_back = sys.exc_info()[2]
raise EngineException(
"PythonLocalExecEngine failed to produce outputs into executable algo [%s]"
% self.executable_algo, err_post).with_traceback(trace_back)
def run(algo_name, arg_names, arg_values, asynchro=False, is_customized_algo=False, run_debug=True):
"""
Launch the algorithm for the specified Implementation/CustomizedAlgo
:param algo_name: the ID is matching an Implementation or a CustomizedAlgo according to the flag is_customized_algo
:type algo_name: str
:param arg_names: list of input names
:type arg_names: list of string
:param arg_values: list of input arguments values matching arg_names
:type arg_values: list
:param is_customized_algo: True if algo_name identifies CustomizedAlgo,
otherwise False if algo_name identifies an Implementation
:type is_customized_algo: boolean
:param run_debug: TODO
:type run_debug: boolean
:return: <status> sums up the execution status:
| You can get from status the process ID, the (error) message etc. : see ExecutionStatus documentation.
:rtype: apps.algo.execute.models.business.exec_status.ExecutionStatus
"""
engine_status = ExecStatus(debug=run_debug, msg="Initial status before starting the execution engine")
execution_status = ExecutionStatus(algo=None, internal_status=engine_status)
my_script_name = "RunAlgo"
msg = "Execute: %s , arg_names=%s" % (my_script_name, str(arg_names))
# status is set to default value
# in case of unexpected error in FacadeExecution.execute below
execution_status.add_msg(msg)
if len(arg_names) != len(arg_values):
msg = 'Argument error in execalgo.run: incorrect arguments number)'
execution_status.set_error_with_msg(IkatsInputError(msg), "RunAlgo: initial check failed")
# CheckEngine status generated when check have been applied
status = None
try:
if is_customized_algo:
my_custom = CustomizedAlgoDao.find_business_elem_with_name(algo_name)[0]
# ... may raise CustomizedAlgoDao.DoesNotExist
my_implementation = my_custom.implementation
else:
my_custom = None
my_implementation = ImplementationDao.find_business_elem_with_name(algo_name)[0]
# ... may raise ImplementationDao.DoesNotExist
my_script_name = my_script_name + " on " + my_implementation.name
def_resource = __get_run_definition(implementation=my_implementation, customized_algo=my_custom)
context = "Run algo with args {} on {}".format(json.dumps(__get_run_args(arg_names, arg_values)),
def_resource)
checker = CheckEngine(checked_resource=def_resource, checked_value_context=context, check_status=None)
completed_arg_names, completed_data_sources, output_names, receivers = __prepare_execution(
algo_name,
arg_names,
arg_values,
execution_status,
my_implementation,
my_custom,
checker)
if checker.has_errors():
# ExecutableAlgo is not created, not executed
raise CheckError(msg=context, status=checker.get_check_status())
if asynchro is True:
# asynchronous execution
# creation of an executable algorithm DAO in order to get an id
exec_algo = FacadeExecution.factory.build_exec_algo_without_custom_without_data_connectors(
my_implementation)
# business object is updated with db_id created by DAO
exec_algo = ExecutableAlgoDao.create(exec_algo, False)
execution_status.set_algo(exec_algo)
exec_algo_db_id = exec_algo.get_process_id()
# Create a thread for the algorithm execution itself
run_msg = "Running asynchronous: {0} with implementation={1}".format(my_script_name, my_implementation.name)
LOGGER.info(run_msg)
execution_status.add_msg(run_msg)
threading.Thread(target=FacadeExecution.execute_algo_without_custom,
args=(my_implementation,
completed_arg_names,
completed_data_sources,
output_names,
receivers,
"" + exec_algo_db_id,
run_debug,
True)).start()
else:
# synchronous execution
run_msg = "Running synchronous: {0} with implementation={1}".format(my_script_name, my_implementation.name)
LOGGER.info(run_msg)
execution_status.add_msg(run_msg)
exec_algo, status = FacadeExecution.execute_algo_without_custom(
implem=my_implementation,
input_arg_names=completed_arg_names,
input_data_sources=completed_data_sources,
output_arg_names=output_names,
output_data_receivers=receivers,
exec_algo_db_id=None,
run_debug=run_debug,
dao_managed=True)
execution_status.set_algo(exec_algo)
# replace local internal status by the engine status: more interesting
execution_status.set_internal_status(status)
run_msg = "Ran synchronous: {0} with implementation={1}".format(my_script_name, my_implementation.name)
execution_status.add_msg(run_msg)
LOGGER.info(run_msg)
except CustomizedAlgoDao.DoesNotExist as cerr:
nf_msg = "Argument error in execalgo.run: unmatched CustomizedAlgo: err={}".format(cerr)
nf_err = IkatsNotFoundError(nf_msg)
execution_status.set_error_with_msg(nf_err, "RunAlgo: initial check failed")
except ImplementationDao.DoesNotExist as ierr:
nf_msg = "Argument error in execalgo.run: unmatched Implementation: err={}".format(ierr)
nf_err = IkatsNotFoundError(nf_msg)
execution_status.set_error_with_msg(nf_err, "RunAlgo: initial check failed")
except (CheckError, IkatsInputError, IkatsNotFoundError, IkatsException) as exception:
# interrupts the executions
# => in that case: executable algo is not created ...
raise exception
except Exception as error:
error_message = "Failure occurred running {0} with implementation={1}".format(
my_script_name, algo_name)
LOGGER.error(error_message)
execution_status.add_msg(error_message)
LOGGER.exception(error)
if (status is None) or (not status.has_error()):
# add error when not yet recorded ...
msg = "Unexpected error occurred"
execution_status.get_state().set_error_with_msg(added_error=error, added_msg=msg, raise_error=False)
execution_status.get_state().debug = True
# logged by calling code
# Note that at this step: execution_status.has_error() may be true
return execution_status
def setUpTestData(cls):
super(TestExecEngineWithCheckEngine, cls).init_tu()
cls.prepare_custom_database()
CheckEngine.set_checking_rules(FactoryCatalogue.TYPE_CHECKING_FUNCTIONS)
cls.request_factory = RequestFactory()