def get_portlet_status(portlet_cfg):
"""
Get the status of a portlet configuration.
"""
if portlet_cfg.analyzers:
for analyze_step in portlet_cfg.analyzers:
operation = dao.try_get_operation_by_id(analyze_step.fk_operation)
if operation is None:
return model.STATUS_ERROR, "Operation has been removed"
if operation.status != model.STATUS_FINISHED:
return operation.status, operation.additional_info or ''
else:
## Simulator is first step so now decide if we are waiting for input or output ##
visualizer = portlet_cfg.visualizer
wait_on_outputs = False
for entry in visualizer.dynamic_param:
if type(visualizer.dynamic_param[entry][WorkflowStepConfiguration.DATATYPE_INDEX_KEY]) == IntType:
wait_on_outputs = True
break
if wait_on_outputs:
simulator_step = dao.get_workflow_step_by_step_index(visualizer.fk_workflow, 0)
operation = dao.try_get_operation_by_id(simulator_step.fk_operation)
if operation is None:
error_msg = ("At least one simulation result was not found, it might have been removed. <br\>"
"You can copy and relaunch current simulation, if you are interested in having "
"your results re-computed.")
return model.STATUS_ERROR, error_msg
else:
return operation.status, operation.additional_info or ''
return model.STATUS_FINISHED, ''
def get_portlet_status(portlet_cfg):
"""
Get the status of a portlet configuration.
"""
if portlet_cfg.analyzers:
for analyze_step in portlet_cfg.analyzers:
operation = dao.try_get_operation_by_id(analyze_step.fk_operation)
if operation is None:
return model.STATUS_ERROR, "Operation has been removed"
if operation.status != model.STATUS_FINISHED:
return operation.status, operation.additional_info or ""
else:
## Simulator is first step so now decide if we are waiting for input or output ##
visualizer = portlet_cfg.visualizer
wait_on_outputs = False
for entry in visualizer.dynamic_param:
if type(visualizer.dynamic_param[entry][WorkflowStepConfiguration.DATATYPE_INDEX_KEY]) == IntType:
wait_on_outputs = True
break
if wait_on_outputs:
simulator_step = dao.get_workflow_step_by_step_index(visualizer.fk_workflow, 0)
operation = dao.try_get_operation_by_id(simulator_step.fk_operation)
if operation is None:
error_msg = (
"At least one simulation result was not found, it might have been removed. <br\>"
"You can copy and relaunch current simulation, if you are interested in having "
"your results re-computed."
)
return model.STATUS_ERROR, error_msg
else:
return operation.status, operation.additional_info or ""
return model.STATUS_FINISHED, ""
def test_remove_burst_operation(self, long_burst_launch):
burst_config = long_burst_launch
operation = self._wait_for_burst_ops(burst_config)[0]
assert not operation.has_finished
self.flow_c.stop_burst_operation(operation.id, 0, True)
operation = dao.try_get_operation_by_id(operation.id)
assert operation is None
def test_remove_burst_operation(self):
burst_config = self._long_burst_launch()
operation = self._wait_for_burst_ops(burst_config)[0]
self.assertFalse(operation.has_finished)
self.flow_c.stop_burst_operation(operation.id, 0, True)
operation = dao.try_get_operation_by_id(operation.id)
self.assertTrue(operation is None)
def stop_operation(operation_id):
"""
Stop a thread for a given operation id
"""
operation = dao.try_get_operation_by_id(operation_id)
if not operation or operation.has_finished:
LOGGER.info("Operation already stopped or not found at ID: %s" % operation_id)
return True
LOGGER.debug("Stopping operation: %s" % str(operation_id))
# Set the thread stop flag to true
for thread in CURRENT_ACTIVE_THREADS:
if int(thread.operation_id) == operation_id:
thread._stop()
LOGGER.debug("Found running thread for operation: %d" % operation_id)
# Kill Thread
stopped = True
operation_process = dao.get_operation_process_for_operation(operation_id)
if operation_process is not None:
# Now try to kill the operation if it exists
stopped = OperationExecutor.stop_pid(operation_process.pid)
if not stopped:
LOGGER.debug("Operation %d was probably killed from it's specific thread." % operation_id)
else:
LOGGER.debug("Stopped OperationExecutor process for %d" % operation_id)
# Mark operation as canceled in DB and on disk
BurstService().persist_operation_state(operation, STATUS_CANCELED)
return stopped
def stop_operation(operation_id):
"""
Stop a thread for a given operation id
"""
operation = dao.try_get_operation_by_id(operation_id)
if not operation or operation.has_finished:
LOGGER.warning("Operation already stopped or not found is given to stop job: %s" % operation_id)
return True
LOGGER.debug("Stopping operation: %s" % str(operation_id))
## Set the thread stop flag to true
for thread in CURRENT_ACTIVE_THREADS:
if int(thread.operation_id) == operation_id:
thread.stop()
LOGGER.debug("Found running thread for operation: %d" % operation_id)
## Kill Thread
stopped = True
operation_process = dao.get_operation_process_for_operation(operation_id)
if operation_process is not None:
## Now try to kill the operation if it exists
stopped = OperationExecutor.stop_pid(operation_process.pid)
if not stopped:
LOGGER.debug("Operation %d was probably killed from it's specific thread." % operation_id)
else:
LOGGER.debug("Stopped OperationExecutor process for %d" % operation_id)
## Mark operation as canceled in DB and on disk
WorkflowService().persist_operation_state(operation, model.STATUS_CANCELED)
return stopped
def test_remove_burst_operation(self):
burst_config = self._long_burst_launch()
operation = self._wait_for_burst_ops(burst_config)[0]
self.assertFalse(operation.has_finished)
self.flow_c.stop_burst_operation(operation.id, 0, True)
operation = dao.try_get_operation_by_id(operation.id)
self.assertTrue(operation is None)
def stop_operation(operation_id, is_group=False, remove_after_stop=False):
# type: (int, bool, bool) -> bool
"""
Stop (also named Cancel) the operation given by operation_id,
and potentially also remove it after (with all linked data).
In case the Operation has a linked Burst, remove that too.
:param operation_id: ID for Operation (or OperationGroup) to be canceled/removed
:param is_group: When true stop all the operations from that group.
:param remove_after_stop: if True, also remove the operation(s) after stopping
:returns True if the stop step was successfully
"""
result = False
if is_group:
op_group = ProjectService.get_operation_group_by_id(operation_id)
operations_in_group = ProjectService.get_operations_in_group(
op_group)
for operation in operations_in_group:
result = OperationService.stop_operation(
operation.id, False, remove_after_stop) or result
elif dao.try_get_operation_by_id(operation_id) is not None:
result = BackendClientFactory.stop_operation(operation_id)
if remove_after_stop:
burst_config = dao.get_burst_for_direct_operation_id(
operation_id)
ProjectService().remove_operation(operation_id)
if burst_config is not None:
result = dao.remove_entity(BurstConfiguration,
burst_config.id) or result
return result
def stop_operation(operation_id):
"""
Stop a thread for a given operation id
"""
operation = dao.try_get_operation_by_id(operation_id)
if not operation or operation.has_finished:
LOGGER.warning(
"Operation already stopped or not found is given to stop job: %s"
% operation_id)
return True
operation_process = dao.get_operation_process_for_operation(
operation_id)
result = 0
# Try to kill only if operation job process is not None
if operation_process is not None:
stop_command = TvbProfile.current.cluster.STOP_COMMAND % operation_process.job_id
LOGGER.info("Stopping cluster operation: %s" % stop_command)
result = os.system(stop_command)
if result != 0:
LOGGER.error(
"Stopping cluster operation was unsuccessful. Try following status with '"
+ TvbProfile.current.cluster.STATUS_COMMAND +
"'" % operation_process.job_id)
WorkflowService().persist_operation_state(operation,
model.STATUS_CANCELED)
return result == 0
def test_remove_burst_operation_group(self, simulation_launch):
first_op = simulation_launch(self.test_user, self.test_project, 1000, True)
operations_group_id = first_op.fk_operation_group
assert not first_op.has_finished
self.flow_c.cancel_or_remove_operation(operations_group_id, 1, True)
operations = dao.get_operations_in_group(operations_group_id)
for operation in operations:
operation = dao.try_get_operation_by_id(operation.id)
assert operation is None
def test_remove_burst_operation_group(self):
burst_config = self._long_burst_launch(True)
operations = self._wait_for_burst_ops(burst_config)
operations_group_id = 0
for operation in operations:
self.assertFalse(operation.has_finished)
operations_group_id = operation.fk_operation_group
self.flow_c.stop_burst_operation(operations_group_id, 1, True)
for operation in operations:
operation = dao.try_get_operation_by_id(operation.id)
self.assertTrue(operation is None)
def test_remove_burst_operation_group(self, long_burst_launch):
burst_config = long_burst_launch(True)
operations = self._wait_for_burst_ops(burst_config)
operations_group_id = 0
for operation in operations:
assert not operation.has_finished
operations_group_id = operation.fk_operation_group
self.flow_c.stop_burst_operation(operations_group_id, 1, True)
for operation in operations:
operation = dao.try_get_operation_by_id(operation.id)
assert operation is None
def remove_operation(self, operation_id):
"""
Remove a given operation
"""
operation = dao.try_get_operation_by_id(operation_id)
if operation is not None:
self.logger.debug("Deleting operation %s " % operation)
datatypes_for_op = dao.get_results_for_operation(operation_id)
for dt in reversed(datatypes_for_op):
self.remove_datatype(operation.project.id, dt.gid, False)
dao.remove_entity(model.Operation, operation.id)
self.logger.debug("Finished deleting operation %s " % operation)
else:
self.logger.warning("Attempt to delete operation with id=%s which no longer exists." % operation_id)
def stop_operation(operation_id):
"""
Stop a thread for a given operation id
"""
operation = dao.try_get_operation_by_id(operation_id)
if not operation or operation.has_finished:
LOGGER.info("Operation already stopped or not found at ID: %s" % operation_id)
return True
LOGGER.debug("Stopping operation: %s" % str(operation_id))
stopped = StandAloneClient.stop_operation_process(operation_id, True)
# Mark operation as canceled in DB and on disk
BurstService().persist_operation_state(operation, STATUS_CANCELED)
return stopped
def remove_operation(self, operation_id):
"""
Remove a given operation
"""
operation = dao.try_get_operation_by_id(operation_id)
if operation is not None:
self.logger.debug("Deleting operation %s " % operation)
datatypes_for_op = dao.get_results_for_operation(operation_id)
for dt in reversed(datatypes_for_op):
self.remove_datatype(operation.project.id, dt.gid, False)
dao.remove_entity(model.Operation, operation.id)
self.logger.debug("Finished deleting operation %s " % operation)
else:
self.logger.warning("Attempt to delete operation with id=%s which no longer exists." % operation_id)
def remove_operation(self, operation_id):
"""
Remove a given operation
"""
operation = dao.try_get_operation_by_id(operation_id)
if operation is not None:
self.logger.debug("Deleting operation %s " % operation)
datatypes_for_op = dao.get_results_for_operation(operation_id)
for dt in reversed(datatypes_for_op):
self.remove_datatype(operation.project.id, dt.gid, False)
# Here the Operation is mot probably already removed - in case DTs were found inside
# but we still remove it for the case when no DTs exist
dao.remove_entity(Operation, operation.id)
self.structure_helper.remove_operation_data(operation.project.name, operation_id)
encryption_handler.push_folder_to_sync(self.structure_helper.get_project_folder(operation.project))
self.logger.debug("Finished deleting operation %s " % operation)
else:
self.logger.warning("Attempt to delete operation with id=%s which no longer exists." % operation_id)
def stop_operation(operation_id):
"""
Stop a thread for a given operation id
"""
operation = dao.try_get_operation_by_id(operation_id)
if not operation or operation.has_finished:
LOGGER.warning("Operation already stopped or not found is given to stop job: %s" % operation_id)
return True
operation_process = dao.get_operation_process_for_operation(operation_id)
result = 0
## Try to kill only if operation job process is not None
if operation_process is not None:
stop_command = TvbProfile.current.cluster.STOP_COMMAND % operation_process.job_id
LOGGER.info("Stopping cluster operation: %s" % stop_command)
result = os.system(stop_command)
if result != 0:
LOGGER.error("Stopping cluster operation was unsuccessful. Try following status with '" +
TvbProfile.current.cluster.STATUS_COMMAND + "'" % operation_process.job_id)
WorkflowService().persist_operation_state(operation, model.STATUS_CANCELED)
return result == 0
def stop_operation(operation_id):
"""
Stop a thread for a given operation id
"""
operation = dao.try_get_operation_by_id(operation_id)
if not operation or operation.has_finished:
LOGGER.warning("Operation already stopped or not found is given to stop job: %s" % operation_id)
return True
operation_process = dao.get_operation_process_for_operation(operation_id)
result = 0
## Try to kill only if operation job process is not None
if operation_process is not None:
stop_command = TvbProfile.current.cluster.STOP_COMMAND % operation_process.job_id
LOGGER.info("Stopping cluster operation: %s" % stop_command)
result = os.system(stop_command)
if result != 0:
LOGGER.error("Stopping cluster operation was unsuccessful. "
"Try following with 'oarstat' for job ID: %s" % operation_process.job_id)
operation.mark_complete(model.STATUS_CANCELED)
dao.store_entity(operation)
return result == 0
def test_remove_burst_operation(self, simulation_launch):
operation = simulation_launch(self.test_user, self.test_project, 1000)
assert not operation.has_finished
self.flow_c.cancel_or_remove_operation(operation.id, 0, True)
operation = dao.try_get_operation_by_id(operation.id)
assert operation is None
