def launch_operation(self, operation_id, send_to_cluster=False, adapter_instance=None): """ Method exposed for Burst-Workflow related calls. It is used for cascading operation in the same workflow. """ if operation_id is not None: operation = dao.get_operation_by_id(operation_id) if adapter_instance is None: algorithm = operation.algorithm adapter_instance = ABCAdapter.build_adapter(algorithm) parsed_params = utils.parse_json_parameters(operation.parameters) if not 'SimulatorAdapter' in adapter_instance.__class__.__name__: adapter_form = adapter_instance.get_form()() adapter_form.fill_from_post(parsed_params) adapter_instance.submit_form(adapter_form) if send_to_cluster: self._send_to_cluster([operation], adapter_instance, operation.user.username) else: self.initiate_prelaunch(operation, adapter_instance, **parsed_params)
def _run_cluster_job(operation_identifier, user_name_label, adapter_instance): """ Threaded Popen It is the function called by the ClusterSchedulerClient in a Thread. This function starts a new process. """ # Load operation so we can estimate the execution time operation = dao.get_operation_by_id(operation_identifier) kwargs = parse_json_parameters(operation.parameters) kwargs = adapter_instance.prepare_ui_inputs(kwargs) time_estimate = int(adapter_instance.get_execution_time_approximation(**kwargs)) hours = int(time_estimate / 3600) minutes = (int(time_estimate) % 3600) / 60 seconds = int(time_estimate) % 60 # Anything lower than 5 hours just use default walltime if hours < 5: walltime = "05:00:00" else: if hours < 10: hours = "0%d" % hours else: hours = str(hours) walltime = "%s:%s:%s" % (hours, str(minutes), str(seconds)) call_arg = TvbProfile.current.cluster.SCHEDULE_COMMAND % (operation_identifier, user_name_label, walltime) LOGGER.info(call_arg) process_ = Popen([call_arg], stdout=PIPE, shell=True) job_id = process_.stdout.read().replace('\n', '').split(TvbProfile.current.cluster.JOB_ID_STRING)[-1] LOGGER.debug("Got jobIdentifier = %s for CLUSTER operationID = %s" % (operation_identifier, job_id)) operation_identifier = model.OperationProcessIdentifier(operation_identifier, job_id=job_id) dao.store_entity(operation_identifier)
def _run_cluster_job(operation_identifier, user_name_label, adapter_instance): """ Threaded Popen It is the function called by the ClusterSchedulerClient in a Thread. This function starts a new process. """ # Load operation so we can estimate the execution time operation = dao.get_operation_by_id(operation_identifier) kwargs = parse_json_parameters(operation.parameters) time_estimate = int(adapter_instance.get_execution_time_approximation(**kwargs)) hours = int(time_estimate / 3600) minutes = (int(time_estimate) % 3600) / 60 seconds = int(time_estimate) % 60 # Anything lower than 2 hours just use default walltime if hours < 2: walltime = "02:00:00" elif hours > 23: walltime = "23:59:59" else: walltime = datetime.time(hours, minutes, seconds) walltime = walltime.strftime("%H:%M:%S") call_arg = config.CLUSTER_SCHEDULE_COMMAND % (walltime, operation_identifier, user_name_label) LOGGER.info(call_arg) process_ = Popen([call_arg], stdout=PIPE, shell=True) job_id = process_.stdout.read().replace('\n', '').split('OAR_JOB_ID=')[-1] LOGGER.debug("Got jobIdentifier = %s for CLUSTER operationID = %s" % (operation_identifier, job_id)) operation_identifier = model.OperationProcessIdentifier(operation_identifier, job_id=job_id) dao.store_entity(operation_identifier)
def prepare_after_load(self): """ Load Simulator configuration from JSON string, as it was stored in DB. """ self.tabs = [TabConfiguration() for _ in range(self.nr_of_tabs)] self.simulator_configuration = parse_json_parameters( self._simulator_configuration)
def do_operation_launch(operation_id): """ Event attached to the local queue for executing an operation, when we will have resources available. """ LOGGER = get_logger('tvb.core.operation_async_launcher') try: LOGGER.debug("Loading operation with id=%s" % operation_id) curent_operation = dao.get_operation_by_id(operation_id) stored_adapter = curent_operation.algorithm LOGGER.debug("Importing Algorithm: " + str(stored_adapter.classname) + " for Operation:" + str(curent_operation.id)) PARAMS = parse_json_parameters(curent_operation.parameters) adapter_instance = ABCAdapter.build_adapter(stored_adapter) ## Un-comment bellow for profiling an operation: ## import cherrypy.lib.profiler as profiler ## p = profiler.Profiler("/Users/lia.domide/TVB/profiler/") ## p.run(OperationService().initiate_prelaunch, curent_operation, adapter_instance, {}, **PARAMS) OperationService().initiate_prelaunch(curent_operation, adapter_instance, {}, **PARAMS) LOGGER.debug("Successfully finished operation " + str(operation_id)) except Exception, excep: LOGGER.error("Could not execute operation " + str(sys.argv[1])) LOGGER.exception(excep) parent_burst = dao.get_burst_for_operation_id(operation_id) if parent_burst is not None: WorkflowService().mark_burst_finished(parent_burst, error_message=str(excep))
def do_operation_launch(operation_id): """ Event attached to the local queue for executing an operation, when we will have resources available. """ LOGGER = get_logger('tvb.core.operation_async_launcher') try: LOGGER.debug("Loading operation with id=%s" % operation_id) curent_operation = dao.get_operation_by_id(operation_id) stored_adapter = curent_operation.algorithm LOGGER.debug("Importing Algorithm: " + str(stored_adapter.classname) + " for Operation:" + str(curent_operation.id)) PARAMS = parse_json_parameters(curent_operation.parameters) adapter_instance = ABCAdapter.build_adapter(stored_adapter) ## Un-comment bellow for profiling an operation: ## import cherrypy.lib.profiler as profiler ## p = profiler.Profiler("/Users/lia.domide/TVB/profiler/") ## p.run(OperationService().initiate_prelaunch, curent_operation, adapter_instance, {}, **PARAMS) OperationService().initiate_prelaunch(curent_operation, adapter_instance, {}, **PARAMS) LOGGER.debug("Successfully finished operation " + str(operation_id)) except Exception as excep: LOGGER.error("Could not execute operation " + str(sys.argv[1])) LOGGER.exception(excep) parent_burst = dao.get_burst_for_operation_id(operation_id) if parent_burst is not None: WorkflowService().mark_burst_finished(parent_burst, error_message=str(excep))
def prepare_after_load(self): """ Load Simulator configuration from JSON string, as it was stored in DB. """ self.tabs = [TabConfiguration() for _ in range(self.nr_of_tabs)] self.simulator_configuration = parse_json_parameters(self._simulator_configuration) self.dynamic_ids = json.loads(self._dynamic_ids)
def _run_cluster_job(operation_identifier, user_name_label, adapter_instance): """ Threaded Popen It is the function called by the ClusterSchedulerClient in a Thread. This function starts a new process. """ # Load operation so we can estimate the execution time operation = dao.get_operation_by_id(operation_identifier) kwargs = parse_json_parameters(operation.parameters) kwargs = adapter_instance.prepare_ui_inputs(kwargs) time_estimate = int(adapter_instance.get_execution_time_approximation(**kwargs)) hours = int(time_estimate / 3600) minutes = (int(time_estimate) % 3600) / 60 seconds = int(time_estimate) % 60 # Anything lower than 5 hours just use default walltime if hours < 5: walltime = "05:00:00" else: if hours < 10: hours = "0%d" % hours else: hours = str(hours) walltime = "%s:%s:%s" % (hours, str(minutes), str(seconds)) call_arg = TvbProfile.current.cluster.SCHEDULE_COMMAND % (walltime, operation_identifier, user_name_label) LOGGER.info(call_arg) process_ = Popen([call_arg], stdout=PIPE, shell=True) job_id = process_.stdout.read().replace('\n', '').split('OAR_JOB_ID=')[-1] LOGGER.debug("Got jobIdentifier = %s for CLUSTER operationID = %s" % (operation_identifier, job_id)) operation_identifier = model.OperationProcessIdentifier(operation_identifier, job_id=job_id) dao.store_entity(operation_identifier)
def do_operation_launch(operation_id): """ Event attached to the local queue for executing an operation, when we will have resources available. """ log = get_logger('tvb.core.operation_async_launcher') burst_service = BurstService2() try: log.debug("Loading operation with id=%s" % operation_id) curent_operation = dao.get_operation_by_id(operation_id) stored_adapter = curent_operation.algorithm log.debug("Importing Algorithm: " + str(stored_adapter.classname) + " for Operation:" + str(curent_operation.id)) params = parse_json_parameters(curent_operation.parameters) adapter_instance = ABCAdapter.build_adapter(stored_adapter) # These should go once we have a common place for it if not isinstance(adapter_instance, SimulatorAdapter): adapter_form = adapter_instance.get_form()( project_id=curent_operation.fk_launched_in) adapter_form.fill_from_post(params) adapter_instance.submit_form(adapter_form) # Un-comment bellow for profiling an operation: # import cherrypy.lib.profiler as profiler # p = profiler.Profiler("/Users/lia.domide/TVB/profiler/") # p.run(OperationService().initiate_prelaunch, curent_operation, adapter_instance, {}, **PARAMS) OperationService().initiate_prelaunch(curent_operation, adapter_instance, **params) if curent_operation.fk_operation_group: parent_burst = dao.get_generic_entity( BurstConfiguration2, curent_operation.fk_operation_group, 'operation_group_id')[0] operations_in_group = dao.get_operations_in_group( curent_operation.fk_operation_group) if parent_burst.metric_operation_group_id: operations_in_group.extend( dao.get_operations_in_group( parent_burst.metric_operation_group_id)) for operation in operations_in_group: if not has_finished(operation.status): break if parent_burst is not None: burst_service.mark_burst_finished(parent_burst) else: parent_burst = burst_service.get_burst_for_operation_id( operation_id) if parent_burst is not None: burst_service.mark_burst_finished(parent_burst) log.debug("Successfully finished operation " + str(operation_id)) except Exception as excep: log.error("Could not execute operation " + str(sys.argv[1])) log.exception(excep) parent_burst = burst_service.get_burst_for_operation_id(operation_id) if parent_burst is not None: burst_service.mark_burst_finished(parent_burst, error_message=str(excep))
def reloadoperation(self, operation_id, **_): """Redirect to Operation Input selection page, with input data already selected.""" operation = self.flow_service.load_operation(operation_id) data = parse_json_parameters(operation.parameters) self.context.add_adapter_to_session(operation.algorithm, None, data) category_id = operation.algorithm.fk_category algo_id = operation.fk_from_algo raise cherrypy.HTTPRedirect("/flow/" + str(category_id) + "/" + str(algo_id) + "?not_reset=True")
def _adapt_epileptor_simulations(): """ Previous Simulations on EpileptorWithPermitivity model, should be converted to use the Epileptor model. As the parameters from the two models are having different ranges and defaults, we do not translate parameters, we only set the Epileptor as model instead of EpileptorPermittivityCoupling, and leave the model params to defaults. """ session = SA_SESSIONMAKER() epileptor_old = "EpileptorPermittivityCoupling" epileptor_new = "Epileptor" param_model = "model" try: all_ep_ops = session.query(model.Operation).filter( model.Operation.parameters.ilike('%"' + epileptor_old + '"%')).all() files_helper = FilesHelper() all_bursts = dict() for ep_op in all_ep_ops: try: op_params = parse_json_parameters(ep_op.parameters) if op_params[param_model] != epileptor_old: LOGGER.debug("Skipping op " + str(op_params[param_model]) + " -- " + str(ep_op)) continue LOGGER.debug("Updating " + str(op_params)) op_params[param_model] = epileptor_new ep_op.parameters = json.dumps(op_params, cls=MapAsJson.MapAsJsonEncoder) LOGGER.debug("New params:" + ep_op.parameters) files_helper.write_operation_metadata(ep_op) burst = dao.get_burst_for_operation_id(ep_op.id) if burst is not None: LOGGER.debug("Updating burst:" + str(burst)) burst.prepare_after_load() burst.simulator_configuration[param_model] = { 'value': epileptor_new } burst._simulator_configuration = json.dumps( burst.simulator_configuration, cls=MapAsJson.MapAsJsonEncoder) if not all_bursts.has_key(burst.id): all_bursts[burst.id] = burst except Exception: LOGGER.exception("Could not process " + str(ep_op)) session.add_all(all_ep_ops) session.add_all(all_bursts.values()) session.commit() except Exception: LOGGER.exception("Could not update Simulation Epileptor Params") finally: session.close()
def _adapt_epileptor_simulations(): """ Previous Simulations on EpileptorWithPermitivity model, should be converted to use the Epileptor model. As the parameters from the two models are having different ranges and defaults, we do not translate parameters, we only set the Epileptor as model instead of EpileptorPermittivityCoupling, and leave the model params to defaults. """ session = SA_SESSIONMAKER() epileptor_old = "EpileptorPermittivityCoupling" epileptor_new = "Epileptor" param_model = "model" try: all_ep_ops = session.query(model.Operation).filter( model.Operation.parameters.ilike('%"' + epileptor_old + '"%')).all() files_helper = FilesHelper() all_bursts = dict() for ep_op in all_ep_ops: try: op_params = parse_json_parameters(ep_op.parameters) if op_params[param_model] != epileptor_old: LOGGER.debug("Skipping op " + str(op_params[param_model]) + " -- " + str(ep_op)) continue LOGGER.debug("Updating " + str(op_params)) op_params[param_model] = epileptor_new ep_op.parameters = json.dumps(op_params, cls=MapAsJson.MapAsJsonEncoder) LOGGER.debug("New params:" + ep_op.parameters) files_helper.write_operation_metadata(ep_op) burst = dao.get_burst_for_operation_id(ep_op.id) if burst is not None: LOGGER.debug("Updating burst:" + str(burst)) burst.prepare_after_load() burst.simulator_configuration[param_model] = {'value': epileptor_new} burst._simulator_configuration = json.dumps(burst.simulator_configuration, cls=MapAsJson.MapAsJsonEncoder) if not all_bursts.has_key(burst.id): all_bursts[burst.id] = burst except Exception: LOGGER.exception("Could not process " + str(ep_op)) session.add_all(all_ep_ops) session.add_all(all_bursts.values()) session.commit() except Exception: LOGGER.exception("Could not update Simulation Epileptor Params") finally: session.close()
def launch_operation(self, operation_id, send_to_cluster=False, adapter_instance=None): """ Method exposed for Burst-Workflow related calls. It is used for cascading operation in the same workflow. """ if operation_id is not None: operation = dao.get_operation_by_id(operation_id) if adapter_instance is None: algorithm = operation.algorithm adapter_instance = ABCAdapter.build_adapter(algorithm) parsed_params = utils.parse_json_parameters(operation.parameters) if send_to_cluster: self._send_to_cluster([operation], adapter_instance, operation.user.username) else: self.initiate_prelaunch(operation, adapter_instance, {}, **parsed_params)
def _adapt_simulation_monitor_params(): """ For previous simulation with EEG monitor, adjust the change of input parameters. """ session = SA_SESSIONMAKER() param_connectivity = "connectivity" param_eeg_proj_old = "monitors_parameters_option_EEG_projection_matrix_data" param_eeg_proj_new = "monitors_parameters_option_EEG_projection" param_eeg_sensors = "monitors_parameters_option_EEG_sensors" param_eeg_rm = "monitors_parameters_option_EEG_region_mapping" try: all_eeg_ops = session.query(model.Operation).filter( model.Operation.parameters.ilike('%"' + param_eeg_proj_old + '"%')).all() files_helper = FilesHelper() all_bursts = dict() for eeg_op in all_eeg_ops: try: op_params = parse_json_parameters(eeg_op.parameters) LOGGER.debug("Updating " + str(op_params)) old_projection_guid = op_params[param_eeg_proj_old] connectivity_guid = op_params[param_connectivity] rm = dao.get_generic_entity(RegionMapping, connectivity_guid, "_connectivity")[0] dt = dao.get_generic_entity(model.DataType, old_projection_guid, "gid")[0] if dt.type == 'ProjectionSurfaceEEG': LOGGER.debug("Previous Prj is surface: " + old_projection_guid) new_projection_guid = old_projection_guid else: new_projection_guid = session.execute(text("""SELECT DT.gid FROM "MAPPED_PROJECTION_MATRIX_DATA" PMO, "DATA_TYPES" DTO, "MAPPED_PROJECTION_MATRIX_DATA" PM, "DATA_TYPES" DT WHERE DTO.id=PMO.id and DT.id=PM.id and PM._sensors=PMO._sensors and PM._sources='""" + rm._surface + """' and DTO.gid='""" + old_projection_guid + """';""")).fetchall()[0][0] LOGGER.debug("New Prj is surface: " + str(new_projection_guid)) sensors_guid = session.execute(text("""SELECT _sensors FROM "MAPPED_PROJECTION_MATRIX_DATA" WHERE id = '""" + str(dt.id) + """';""")).fetchall()[0][0] del op_params[param_eeg_proj_old] op_params[param_eeg_proj_new] = str(new_projection_guid) op_params[param_eeg_sensors] = str(sensors_guid) op_params[param_eeg_rm] = str(rm.gid) eeg_op.parameters = json.dumps(op_params, cls=MapAsJson.MapAsJsonEncoder) LOGGER.debug("New params:" + eeg_op.parameters) files_helper.write_operation_metadata(eeg_op) burst = dao.get_burst_for_operation_id(eeg_op.id) if burst is not None: LOGGER.debug("Updating burst:" + str(burst)) burst.prepare_after_load() del burst.simulator_configuration[param_eeg_proj_old] burst.simulator_configuration[param_eeg_proj_new] = {'value': str(new_projection_guid)} burst.simulator_configuration[param_eeg_sensors] = {'value': str(sensors_guid)} burst.simulator_configuration[param_eeg_rm] = {'value': str(rm.gid)} burst._simulator_configuration = json.dumps(burst.simulator_configuration, cls=MapAsJson.MapAsJsonEncoder) if burst.id not in all_bursts: all_bursts[burst.id] = burst except Exception: LOGGER.exception("Could not process " + str(eeg_op)) session.add_all(all_eeg_ops) session.add_all(list(all_bursts.values())) session.commit() except Exception: LOGGER.exception("Could not update Simulation Monitor Params") finally: session.close()
def _adapt_simulation_monitor_params(): """ For previous simulation with EEG monitor, adjust the change of input parameters. """ session = SA_SESSIONMAKER() param_connectivity = "connectivity" param_eeg_proj_old = "monitors_parameters_option_EEG_projection_matrix_data" param_eeg_proj_new = "monitors_parameters_option_EEG_projection" param_eeg_sensors = "monitors_parameters_option_EEG_sensors" param_eeg_rm = "monitors_parameters_option_EEG_region_mapping" try: all_eeg_ops = session.query(model.Operation).filter( model.Operation.parameters.ilike('%"' + param_eeg_proj_old + '"%')).all() files_helper = FilesHelper() all_bursts = dict() for eeg_op in all_eeg_ops: try: op_params = parse_json_parameters(eeg_op.parameters) LOGGER.debug("Updating " + str(op_params)) old_projection_guid = op_params[param_eeg_proj_old] connectivity_guid = op_params[param_connectivity] rm = dao.get_generic_entity(RegionMapping, connectivity_guid, "_connectivity")[0] dt = dao.get_generic_entity(model.DataType, old_projection_guid, "gid")[0] if dt.type == 'ProjectionSurfaceEEG': LOGGER.debug("Previous Prj is surface: " + old_projection_guid) new_projection_guid = old_projection_guid else: new_projection_guid = session.execute(text("""SELECT DT.gid FROM "MAPPED_PROJECTION_MATRIX_DATA" PMO, "DATA_TYPES" DTO, "MAPPED_PROJECTION_MATRIX_DATA" PM, "DATA_TYPES" DT WHERE DTO.id=PMO.id and DT.id=PM.id and PM._sensors=PMO._sensors and PM._sources='""" + rm._surface + """' and DTO.gid='""" + old_projection_guid + """';""")).fetchall()[0][0] LOGGER.debug("New Prj is surface: " + str(new_projection_guid)) sensors_guid = session.execute(text("""SELECT _sensors FROM "MAPPED_PROJECTION_MATRIX_DATA" WHERE id = '""" + str(dt.id) + """';""")).fetchall()[0][0] del op_params[param_eeg_proj_old] op_params[param_eeg_proj_new] = str(new_projection_guid) op_params[param_eeg_sensors] = str(sensors_guid) op_params[param_eeg_rm] = str(rm.gid) eeg_op.parameters = json.dumps(op_params, cls=MapAsJson.MapAsJsonEncoder) LOGGER.debug("New params:" + eeg_op.parameters) files_helper.write_operation_metadata(eeg_op) burst = dao.get_burst_for_operation_id(eeg_op.id) if burst is not None: LOGGER.debug("Updating burst:" + str(burst)) burst.prepare_after_load() del burst.simulator_configuration[param_eeg_proj_old] burst.simulator_configuration[param_eeg_proj_new] = {'value': str(new_projection_guid)} burst.simulator_configuration[param_eeg_sensors] = {'value': str(sensors_guid)} burst.simulator_configuration[param_eeg_rm] = {'value': str(rm.gid)} burst._simulator_configuration = json.dumps(burst.simulator_configuration, cls=MapAsJson.MapAsJsonEncoder) if not all_bursts.has_key(burst.id): all_bursts[burst.id] = burst except Exception: LOGGER.exception("Could not process " + str(eeg_op)) session.add_all(all_eeg_ops) session.add_all(all_bursts.values()) session.commit() except Exception: LOGGER.exception("Could not update Simulation Monitor Params") finally: session.close()