def test_seq3_read():
"""
Tests the read of Algorithm execution status
"""
# even if unused: create stub data for Implementation
my_impl_fake = Implementation(name="TU Fake Impl", description="TU fake",
execution_plugin="TU fake",
library_address="TU Fake",
input_profile=None,
output_profile=None,
db_id=None)
# even if unused: create stub data for CustomizedAlgo
my_custom_algo_fake = CustomizedAlgo(arg_implementation=my_impl_fake, custom_params=None)
current_date = time.time()
LOGGER.info("Current date %s", current_date)
my_exec_algo = ExecutableAlgo(
custom_algo=my_custom_algo_fake, dict_data_sources={}, dict_data_receivers={}, arg_process_id=None)
my_exec_algo.trigger_start_execution_date()
my_exec_algo.trigger_end_execution_date()
my_exec_algo.state = EnumState.ALGO_OK
# ignore unsaved data here : we test the update in database
my_exec_algo_created = ExecutableAlgoDao.create(my_exec_algo, merge_with_unsaved_data=False)
string_algo_before = my_exec_algo_created.as_detailed_string()
LOGGER.info("DB Before read : %s", string_algo_before)
my_exec_algo_read = ExecutableAlgoDao.find_from_key(my_exec_algo_created.process_id)
string_db_read = my_exec_algo_read.as_detailed_string()
LOGGER.info("DB after read : %s", string_db_read)
def get_algo_db(process_id):
"""
returns data stored in DB about any algorithm whose execution has been previously launched
:param process_id: executable algorithm identifier
:type process_id: str
:return: executable algorithm from DB
:rtype: apps.algo.execute.models.business.algo.ExecutableAlgo
"""
my_algo = ExecutableAlgoDao.find_from_key(process_id)
if my_algo is None:
raise IkatsNotFoundError(
"Resource not found in scripts.execalgo.get_algo_db: ExecutableAlgo with id=" + process_id)
return my_algo
def test_seq1_create(self):
"""
Tests the creation of Algorithm execution
"""
# even if unused: create stub data for Implementation
my_impl_fake = Implementation(name="TU Fake Impl", description="TU fake",
execution_plugin="TU fake",
library_address="TU Fake",
input_profile=None,
output_profile=None,
db_id=None)
# even if unused: create stub data for CustomizedAlgo
my_custom_algo_fake = CustomizedAlgo(arg_implementation=my_impl_fake, custom_params=None)
current_date = time.time()
LOGGER.info("Current date %s", current_date)
my_exec_algo = ExecutableAlgo(
custom_algo=my_custom_algo_fake, dict_data_sources={}, dict_data_receivers={}, arg_process_id=None)
string_algo_before = my_exec_algo.as_detailed_string()
LOGGER.info("creation date= %s", my_exec_algo.creation_date)
nb_init = ExecutableAlgoDao.objects.count()
LOGGER.info("After test_seq1_create: ImplementationDao count = %s", nb_init)
my_exec_algo_created = ExecutableAlgoDao.create(my_exec_algo, merge_with_unsaved_data=True)
string_algo_after = my_exec_algo_created.as_detailed_string()
LOGGER.info("Before creation: %s", string_algo_before)
LOGGER.info("After creation: %s", string_algo_after)
my_exec_algo_created.trigger_start_execution_date()
LOGGER.info("Triggered start: %s", my_exec_algo_created.as_detailed_string())
my_exec_algo_created.trigger_end_execution_date()
LOGGER.info("Triggered end : %s", my_exec_algo_created.as_detailed_string())
my_exec_algo_created.state = EnumState.ALGO_KO
string_algo_state_updated = my_exec_algo_created.as_detailed_string()
LOGGER.info("State updated : %s", string_algo_state_updated)
nb_after = ExecutableAlgoDao.objects.count()
self.assertEqual(nb_init + 1, nb_after, "count in DB: expected: nb_init + 1 == nb_after")
def __init__(self, executable_algo, debug=False, dao_managed=False):
assert (isinstance(executable_algo, ExecutableAlgo))
if executable_algo.state != EnumState.INIT:
raise ValueError("Unexpected: executable algo must be in state INIT")
self.__dao_managed = dao_managed
# create exec algo in database, if required
#
self.__executable_algo = executable_algo
if dao_managed:
if executable_algo.process_id is None:
# Not yet save !
self.__executable_algo = ExecutableAlgoDao.create(
original_business_obj=executable_algo, merge_with_unsaved_data=True)
self.__status = ExecStatus(debug=debug)
def run(algo_name, arg_names, arg_values, asynchro=False, is_customized_algo=False, run_debug=True):
"""
Launch the algorithm for the specified Implementation/CustomizedAlgo
:param algo_name: the ID is matching an Implementation or a CustomizedAlgo according to the flag is_customized_algo
:type algo_name: str
:param arg_names: list of input names
:type arg_names: list of string
:param arg_values: list of input arguments values matching arg_names
:type arg_values: list
:param is_customized_algo: True if algo_name identifies CustomizedAlgo,
otherwise False if algo_name identifies an Implementation
:type is_customized_algo: boolean
:param run_debug: TODO
:type run_debug: boolean
:return: <status> sums up the execution status:
| You can get from status the process ID, the (error) message etc. : see ExecutionStatus documentation.
:rtype: apps.algo.execute.models.business.exec_status.ExecutionStatus
"""
engine_status = ExecStatus(debug=run_debug, msg="Initial status before starting the execution engine")
execution_status = ExecutionStatus(algo=None, internal_status=engine_status)
my_script_name = "RunAlgo"
msg = "Execute: %s , arg_names=%s" % (my_script_name, str(arg_names))
# status is set to default value
# in case of unexpected error in FacadeExecution.execute below
execution_status.add_msg(msg)
if len(arg_names) != len(arg_values):
msg = 'Argument error in execalgo.run: incorrect arguments number)'
execution_status.set_error_with_msg(IkatsInputError(msg), "RunAlgo: initial check failed")
# CheckEngine status generated when check have been applied
status = None
try:
if is_customized_algo:
my_custom = CustomizedAlgoDao.find_business_elem_with_name(algo_name)[0]
# ... may raise CustomizedAlgoDao.DoesNotExist
my_implementation = my_custom.implementation
else:
my_custom = None
my_implementation = ImplementationDao.find_business_elem_with_name(algo_name)[0]
# ... may raise ImplementationDao.DoesNotExist
my_script_name = my_script_name + " on " + my_implementation.name
def_resource = __get_run_definition(implementation=my_implementation, customized_algo=my_custom)
context = "Run algo with args {} on {}".format(json.dumps(__get_run_args(arg_names, arg_values)),
def_resource)
checker = CheckEngine(checked_resource=def_resource, checked_value_context=context, check_status=None)
completed_arg_names, completed_data_sources, output_names, receivers = __prepare_execution(
algo_name,
arg_names,
arg_values,
execution_status,
my_implementation,
my_custom,
checker)
if checker.has_errors():
# ExecutableAlgo is not created, not executed
raise CheckError(msg=context, status=checker.get_check_status())
if asynchro is True:
# asynchronous execution
# creation of an executable algorithm DAO in order to get an id
exec_algo = FacadeExecution.factory.build_exec_algo_without_custom_without_data_connectors(
my_implementation)
# business object is updated with db_id created by DAO
exec_algo = ExecutableAlgoDao.create(exec_algo, False)
execution_status.set_algo(exec_algo)
exec_algo_db_id = exec_algo.get_process_id()
# Create a thread for the algorithm execution itself
run_msg = "Running asynchronous: {0} with implementation={1}".format(my_script_name, my_implementation.name)
LOGGER.info(run_msg)
execution_status.add_msg(run_msg)
threading.Thread(target=FacadeExecution.execute_algo_without_custom,
args=(my_implementation,
completed_arg_names,
completed_data_sources,
output_names,
receivers,
"" + exec_algo_db_id,
run_debug,
True)).start()
else:
# synchronous execution
run_msg = "Running synchronous: {0} with implementation={1}".format(my_script_name, my_implementation.name)
LOGGER.info(run_msg)
execution_status.add_msg(run_msg)
exec_algo, status = FacadeExecution.execute_algo_without_custom(
implem=my_implementation,
input_arg_names=completed_arg_names,
input_data_sources=completed_data_sources,
output_arg_names=output_names,
output_data_receivers=receivers,
exec_algo_db_id=None,
run_debug=run_debug,
dao_managed=True)
execution_status.set_algo(exec_algo)
# replace local internal status by the engine status: more interesting
execution_status.set_internal_status(status)
run_msg = "Ran synchronous: {0} with implementation={1}".format(my_script_name, my_implementation.name)
execution_status.add_msg(run_msg)
LOGGER.info(run_msg)
except CustomizedAlgoDao.DoesNotExist as cerr:
nf_msg = "Argument error in execalgo.run: unmatched CustomizedAlgo: err={}".format(cerr)
nf_err = IkatsNotFoundError(nf_msg)
execution_status.set_error_with_msg(nf_err, "RunAlgo: initial check failed")
except ImplementationDao.DoesNotExist as ierr:
nf_msg = "Argument error in execalgo.run: unmatched Implementation: err={}".format(ierr)
nf_err = IkatsNotFoundError(nf_msg)
execution_status.set_error_with_msg(nf_err, "RunAlgo: initial check failed")
except (CheckError, IkatsInputError, IkatsNotFoundError, IkatsException) as exception:
# interrupts the executions
# => in that case: executable algo is not created ...
raise exception
except Exception as error:
error_message = "Failure occurred running {0} with implementation={1}".format(
my_script_name, algo_name)
LOGGER.error(error_message)
execution_status.add_msg(error_message)
LOGGER.exception(error)
if (status is None) or (not status.has_error()):
# add error when not yet recorded ...
msg = "Unexpected error occurred"
execution_status.get_state().set_error_with_msg(added_error=error, added_msg=msg, raise_error=False)
execution_status.get_state().debug = True
# logged by calling code
# Note that at this step: execution_status.has_error() may be true
return execution_status
def execute(self):
"""
This method is published as the entry point of execution for the client:
it internally call run_command and updates finally the returned self.status
:return: self.status
"""
try:
LOGGER.info("ExecEngine::execute update process_id on each data sources and data receivers")
self.__status.add_msg("update process_id on each data sources and data receivers")
self.__executable_algo.update_connectors_with_pid()
LOGGER.info("ExecEngine::execute started: %s", self.__executable_algo.as_detailed_string())
self.__status.add_msg("update executable algo: trigger start date + RUN")
self.__executable_algo.trigger_start_execution_date()
self.__executable_algo.state = EnumState.RUN
if self.__dao_managed:
self.__executable_algo = ExecutableAlgoDao.update(self.__executable_algo,
merge_with_unsaved_data=True)
LOGGER.debug("ExecEngine::run_command() ... ")
self.__status.add_msg("run_command ...")
self.run_command()
LOGGER.debug("... run_command() finished without exception")
self.__status.add_msg("sets on exec algo: state=OK")
self.__executable_algo.state = EnumState.ALGO_OK
if self.__dao_managed:
self.__executable_algo = ExecutableAlgoDao.update(self.__executable_algo,
merge_with_unsaved_data=True)
except AlgoException as err_alg:
trace_back = sys.exc_info()[2]
stack_list = traceback.format_tb(trace_back, limit=None)
LOGGER.error("Caught AlgoException: ")
LOGGER.exception(err_alg)
self.status.error = err_alg
self.status.add_msg("Got error raised by algorithm => ExecutableAlgo.state == ALGO_KO")
self.status.add_msg(str(err_alg))
if stack_list and (len(stack_list) > 0):
for line_error in stack_list:
self.status.add_msg(line_error)
self.__executable_algo.state = EnumState.ALGO_KO
except EngineException as err_engine:
LOGGER.error("EngineException: ")
LOGGER.exception(err_engine)
self.status.error = err_engine
self.status.add_msg("Got error raised by engine => ExecutableAlgo.state == ENGINE_KO")
self.__executable_algo.state = EnumState.ENGINE_KO
except Exception as e:
LOGGER.error("Exception: ")
LOGGER.exception(e)
finally:
self.__executable_algo.trigger_end_execution_date()
if (not ((self.__executable_algo.end_execution_date is None) or
(self.__executable_algo.start_execution_date is None))):
LOGGER.info("- execute() handled in %s sec.",
self.__executable_algo.end_execution_date - self.__executable_algo.start_execution_date)
LOGGER.info("ExecEngine::execute ... ended: %s", self.__executable_algo.as_detailed_string())
if self.__dao_managed:
try:
self.__executable_algo = ExecutableAlgoDao.update(self.__executable_algo,
merge_with_unsaved_data=True)
except Exception as save_err:
LOGGER.error("EngineException: saving the ExecutableAlgo in DB")
LOGGER.exception(save_err)
if self.__executable_algo.state == EnumState.RUN:
my_msg = "Algo has been run with success, producing results; \
but the ExecutableAlgoDao.update(...) failed. Please contact administrator."
self.status.add_msg(my_msg)
LOGGER.error(my_msg)
self.status.error = save_err
self.__executable_algo.state = EnumState.ENGINE_KO
return self.status