def get_portlet_status(portlet_cfg): """ Get the status of a portlet configuration. """ if portlet_cfg.analyzers: for analyze_step in portlet_cfg.analyzers: operation = dao.try_get_operation_by_id(analyze_step.fk_operation) if operation is None: return model.STATUS_ERROR, "Operation has been removed" if operation.status != model.STATUS_FINISHED: return operation.status, operation.additional_info or '' else: ## Simulator is first step so now decide if we are waiting for input or output ## visualizer = portlet_cfg.visualizer wait_on_outputs = False for entry in visualizer.dynamic_param: if type(visualizer.dynamic_param[entry][WorkflowStepConfiguration.DATATYPE_INDEX_KEY]) == IntType: wait_on_outputs = True break if wait_on_outputs: simulator_step = dao.get_workflow_step_by_step_index(visualizer.fk_workflow, 0) operation = dao.try_get_operation_by_id(simulator_step.fk_operation) if operation is None: error_msg = ("At least one simulation result was not found, it might have been removed. <br\>" "You can copy and relaunch current simulation, if you are interested in having " "your results re-computed.") return model.STATUS_ERROR, error_msg else: return operation.status, operation.additional_info or '' return model.STATUS_FINISHED, ''
def get_portlet_status(portlet_cfg): """ Get the status of a portlet configuration. """ if portlet_cfg.analyzers: for analyze_step in portlet_cfg.analyzers: operation = dao.try_get_operation_by_id(analyze_step.fk_operation) if operation is None: return model.STATUS_ERROR, "Operation has been removed" if operation.status != model.STATUS_FINISHED: return operation.status, operation.additional_info or "" else: ## Simulator is first step so now decide if we are waiting for input or output ## visualizer = portlet_cfg.visualizer wait_on_outputs = False for entry in visualizer.dynamic_param: if type(visualizer.dynamic_param[entry][WorkflowStepConfiguration.DATATYPE_INDEX_KEY]) == IntType: wait_on_outputs = True break if wait_on_outputs: simulator_step = dao.get_workflow_step_by_step_index(visualizer.fk_workflow, 0) operation = dao.try_get_operation_by_id(simulator_step.fk_operation) if operation is None: error_msg = ( "At least one simulation result was not found, it might have been removed. <br\>" "You can copy and relaunch current simulation, if you are interested in having " "your results re-computed." ) return model.STATUS_ERROR, error_msg else: return operation.status, operation.additional_info or "" return model.STATUS_FINISHED, ""
def test_remove_burst_operation(self, long_burst_launch): burst_config = long_burst_launch operation = self._wait_for_burst_ops(burst_config)[0] assert not operation.has_finished self.flow_c.stop_burst_operation(operation.id, 0, True) operation = dao.try_get_operation_by_id(operation.id) assert operation is None
def test_remove_burst_operation(self): burst_config = self._long_burst_launch() operation = self._wait_for_burst_ops(burst_config)[0] self.assertFalse(operation.has_finished) self.flow_c.stop_burst_operation(operation.id, 0, True) operation = dao.try_get_operation_by_id(operation.id) self.assertTrue(operation is None)
def stop_operation(operation_id): """ Stop a thread for a given operation id """ operation = dao.try_get_operation_by_id(operation_id) if not operation or operation.has_finished: LOGGER.info("Operation already stopped or not found at ID: %s" % operation_id) return True LOGGER.debug("Stopping operation: %s" % str(operation_id)) # Set the thread stop flag to true for thread in CURRENT_ACTIVE_THREADS: if int(thread.operation_id) == operation_id: thread._stop() LOGGER.debug("Found running thread for operation: %d" % operation_id) # Kill Thread stopped = True operation_process = dao.get_operation_process_for_operation(operation_id) if operation_process is not None: # Now try to kill the operation if it exists stopped = OperationExecutor.stop_pid(operation_process.pid) if not stopped: LOGGER.debug("Operation %d was probably killed from it's specific thread." % operation_id) else: LOGGER.debug("Stopped OperationExecutor process for %d" % operation_id) # Mark operation as canceled in DB and on disk BurstService().persist_operation_state(operation, STATUS_CANCELED) return stopped
def stop_operation(operation_id): """ Stop a thread for a given operation id """ operation = dao.try_get_operation_by_id(operation_id) if not operation or operation.has_finished: LOGGER.warning("Operation already stopped or not found is given to stop job: %s" % operation_id) return True LOGGER.debug("Stopping operation: %s" % str(operation_id)) ## Set the thread stop flag to true for thread in CURRENT_ACTIVE_THREADS: if int(thread.operation_id) == operation_id: thread.stop() LOGGER.debug("Found running thread for operation: %d" % operation_id) ## Kill Thread stopped = True operation_process = dao.get_operation_process_for_operation(operation_id) if operation_process is not None: ## Now try to kill the operation if it exists stopped = OperationExecutor.stop_pid(operation_process.pid) if not stopped: LOGGER.debug("Operation %d was probably killed from it's specific thread." % operation_id) else: LOGGER.debug("Stopped OperationExecutor process for %d" % operation_id) ## Mark operation as canceled in DB and on disk WorkflowService().persist_operation_state(operation, model.STATUS_CANCELED) return stopped
def stop_operation(operation_id, is_group=False, remove_after_stop=False): # type: (int, bool, bool) -> bool """ Stop (also named Cancel) the operation given by operation_id, and potentially also remove it after (with all linked data). In case the Operation has a linked Burst, remove that too. :param operation_id: ID for Operation (or OperationGroup) to be canceled/removed :param is_group: When true stop all the operations from that group. :param remove_after_stop: if True, also remove the operation(s) after stopping :returns True if the stop step was successfully """ result = False if is_group: op_group = ProjectService.get_operation_group_by_id(operation_id) operations_in_group = ProjectService.get_operations_in_group( op_group) for operation in operations_in_group: result = OperationService.stop_operation( operation.id, False, remove_after_stop) or result elif dao.try_get_operation_by_id(operation_id) is not None: result = BackendClientFactory.stop_operation(operation_id) if remove_after_stop: burst_config = dao.get_burst_for_direct_operation_id( operation_id) ProjectService().remove_operation(operation_id) if burst_config is not None: result = dao.remove_entity(BurstConfiguration, burst_config.id) or result return result
def stop_operation(operation_id): """ Stop a thread for a given operation id """ operation = dao.try_get_operation_by_id(operation_id) if not operation or operation.has_finished: LOGGER.warning( "Operation already stopped or not found is given to stop job: %s" % operation_id) return True operation_process = dao.get_operation_process_for_operation( operation_id) result = 0 # Try to kill only if operation job process is not None if operation_process is not None: stop_command = TvbProfile.current.cluster.STOP_COMMAND % operation_process.job_id LOGGER.info("Stopping cluster operation: %s" % stop_command) result = os.system(stop_command) if result != 0: LOGGER.error( "Stopping cluster operation was unsuccessful. Try following status with '" + TvbProfile.current.cluster.STATUS_COMMAND + "'" % operation_process.job_id) WorkflowService().persist_operation_state(operation, model.STATUS_CANCELED) return result == 0
def test_remove_burst_operation_group(self, simulation_launch): first_op = simulation_launch(self.test_user, self.test_project, 1000, True) operations_group_id = first_op.fk_operation_group assert not first_op.has_finished self.flow_c.cancel_or_remove_operation(operations_group_id, 1, True) operations = dao.get_operations_in_group(operations_group_id) for operation in operations: operation = dao.try_get_operation_by_id(operation.id) assert operation is None
def test_remove_burst_operation_group(self): burst_config = self._long_burst_launch(True) operations = self._wait_for_burst_ops(burst_config) operations_group_id = 0 for operation in operations: self.assertFalse(operation.has_finished) operations_group_id = operation.fk_operation_group self.flow_c.stop_burst_operation(operations_group_id, 1, True) for operation in operations: operation = dao.try_get_operation_by_id(operation.id) self.assertTrue(operation is None)
def test_remove_burst_operation_group(self, long_burst_launch): burst_config = long_burst_launch(True) operations = self._wait_for_burst_ops(burst_config) operations_group_id = 0 for operation in operations: assert not operation.has_finished operations_group_id = operation.fk_operation_group self.flow_c.stop_burst_operation(operations_group_id, 1, True) for operation in operations: operation = dao.try_get_operation_by_id(operation.id) assert operation is None
def remove_operation(self, operation_id): """ Remove a given operation """ operation = dao.try_get_operation_by_id(operation_id) if operation is not None: self.logger.debug("Deleting operation %s " % operation) datatypes_for_op = dao.get_results_for_operation(operation_id) for dt in reversed(datatypes_for_op): self.remove_datatype(operation.project.id, dt.gid, False) dao.remove_entity(model.Operation, operation.id) self.logger.debug("Finished deleting operation %s " % operation) else: self.logger.warning("Attempt to delete operation with id=%s which no longer exists." % operation_id)
def stop_operation(operation_id): """ Stop a thread for a given operation id """ operation = dao.try_get_operation_by_id(operation_id) if not operation or operation.has_finished: LOGGER.info("Operation already stopped or not found at ID: %s" % operation_id) return True LOGGER.debug("Stopping operation: %s" % str(operation_id)) stopped = StandAloneClient.stop_operation_process(operation_id, True) # Mark operation as canceled in DB and on disk BurstService().persist_operation_state(operation, STATUS_CANCELED) return stopped
def remove_operation(self, operation_id): """ Remove a given operation """ operation = dao.try_get_operation_by_id(operation_id) if operation is not None: self.logger.debug("Deleting operation %s " % operation) datatypes_for_op = dao.get_results_for_operation(operation_id) for dt in reversed(datatypes_for_op): self.remove_datatype(operation.project.id, dt.gid, False) # Here the Operation is mot probably already removed - in case DTs were found inside # but we still remove it for the case when no DTs exist dao.remove_entity(Operation, operation.id) self.structure_helper.remove_operation_data(operation.project.name, operation_id) encryption_handler.push_folder_to_sync(self.structure_helper.get_project_folder(operation.project)) self.logger.debug("Finished deleting operation %s " % operation) else: self.logger.warning("Attempt to delete operation with id=%s which no longer exists." % operation_id)
def stop_operation(operation_id): """ Stop a thread for a given operation id """ operation = dao.try_get_operation_by_id(operation_id) if not operation or operation.has_finished: LOGGER.warning("Operation already stopped or not found is given to stop job: %s" % operation_id) return True operation_process = dao.get_operation_process_for_operation(operation_id) result = 0 ## Try to kill only if operation job process is not None if operation_process is not None: stop_command = TvbProfile.current.cluster.STOP_COMMAND % operation_process.job_id LOGGER.info("Stopping cluster operation: %s" % stop_command) result = os.system(stop_command) if result != 0: LOGGER.error("Stopping cluster operation was unsuccessful. Try following status with '" + TvbProfile.current.cluster.STATUS_COMMAND + "'" % operation_process.job_id) WorkflowService().persist_operation_state(operation, model.STATUS_CANCELED) return result == 0
def stop_operation(operation_id): """ Stop a thread for a given operation id """ operation = dao.try_get_operation_by_id(operation_id) if not operation or operation.has_finished: LOGGER.warning("Operation already stopped or not found is given to stop job: %s" % operation_id) return True operation_process = dao.get_operation_process_for_operation(operation_id) result = 0 ## Try to kill only if operation job process is not None if operation_process is not None: stop_command = TvbProfile.current.cluster.STOP_COMMAND % operation_process.job_id LOGGER.info("Stopping cluster operation: %s" % stop_command) result = os.system(stop_command) if result != 0: LOGGER.error("Stopping cluster operation was unsuccessful. " "Try following with 'oarstat' for job ID: %s" % operation_process.job_id) operation.mark_complete(model.STATUS_CANCELED) dao.store_entity(operation) return result == 0
def test_remove_burst_operation(self, simulation_launch): operation = simulation_launch(self.test_user, self.test_project, 1000) assert not operation.has_finished self.flow_c.cancel_or_remove_operation(operation.id, 0, True) operation = dao.try_get_operation_by_id(operation.id) assert operation is None