class WorkflowTest(TransactionalTestCase): """ Test that workflow conversion methods are valid. """ def setUp(self): # self.clean_database() self.test_user = TestFactory.create_user() self.test_project = TestFactory.create_project(self.test_user) self.old_config_file = cfg.CURRENT_DIR cfg.CURRENT_DIR = os.path.dirname(tvb_test.__file__) self.workflow_service = WorkflowService() self.burst_service = BurstService() self.operation_service = OperationService() self.flow_service = FlowService() def tearDown(self): """ Remove project folders and clean up database. """ FilesHelper().remove_project_structure(self.test_project.name) self.delete_project_folders() cfg.CURRENT_DIR = self.old_config_file def __create_complex_workflow(self, workflow_step_list): """ Creates a burst with a complex workflow with a given list of workflow steps. @param workflow_step_list: a lsit of workflow steps that will be used in the creation of a new workflow for a new burst """ burst_config = TestFactory.store_burst(self.test_project.id) stored_dt = datatypes_factory.DatatypesFactory()._store_datatype( Datatype1()) first_step_algorithm = self.flow_service.get_algorithm_by_module_and_class( "tvb_test.adapters.testadapter1", "TestAdapterDatatypeInput")[0] metadata = {DataTypeMetaData.KEY_BURST: burst_config.id} kwargs = {"test_dt_input": stored_dt.gid, 'test_non_dt_input': '0'} operations, group = self.operation_service.prepare_operations( self.test_user.id, self.test_project.id, first_step_algorithm, first_step_algorithm.algo_group.group_category, metadata, **kwargs) workflows = self.workflow_service.create_and_store_workflow( project_id=self.test_project.id, burst_id=burst_config.id, simulator_index=0, simulator_id=first_step_algorithm.id, operations=operations) self.operation_service.prepare_operations_for_workflowsteps( workflow_step_list, workflows, self.test_user.id, burst_config.id, self.test_project.id, group, operations) #fire the first op if len(operations) > 0: self.operation_service.launch_operation(operations[0].id, False) return burst_config.id def test_workflow_generation(self): """ A simple test just for the fact that a workflow is created an ran, no dynamic parameters are passed. In this case we create a two steps workflow: step1 - tvb_test.adapters.testadapter2.TestAdapter2 step2 - tvb_test.adapters.testadapter1.TestAdapter1 The first adapter doesn't return anything and the second returns one tvb.datatypes.datatype1.Datatype1 instance. We check that the steps are actually ran by checking that two operations are created and that one dataType is stored. """ workflow_step_list = [ TestFactory.create_workflow_step("tvb_test.adapters.testadapter2", "TestAdapter2", static_kwargs={"test2": 2}, step_index=1), TestFactory.create_workflow_step("tvb_test.adapters.testadapter1", "TestAdapter1", static_kwargs={ "test1_val1": 1, "test1_val2": 1 }, step_index=2) ] self.__create_complex_workflow(workflow_step_list) stored_datatypes = dao.get_datatypes_info_for_project( self.test_project.id) self.assertTrue( len(stored_datatypes) == 2, "DataType from second step was not stored.") self.assertTrue(stored_datatypes[0][0] == 'Datatype1', "Wrong type was stored.") self.assertTrue(stored_datatypes[1][0] == 'Datatype1', "Wrong type was stored.") finished, started, error, _ = dao.get_operation_numbers( self.test_project.id) self.assertEqual( finished, 3, "Didnt start operations for both adapters in workflow.") self.assertEqual(started, 0, "Some operations from workflow didnt finish.") self.assertEqual(error, 0, "Some operations finished with error status.") def test_workflow_dynamic_params(self): """ A simple test just for the fact that dynamic parameters are passed properly between two workflow steps: step1 - tvb_test.adapters.testadapter1.TestAdapter1 step2 - tvb_test.adapters.testadapter3.TestAdapter3 The first adapter returns a tvb.datatypes.datatype1.Datatype1 instance. The second adapter has this passed as a dynamic workflow parameter. We check that the steps are actually ran by checking that two operations are created and that two dataTypes are stored. """ workflow_step_list = [ TestFactory.create_workflow_step("tvb_test.adapters.testadapter1", "TestAdapter1", static_kwargs={ "test1_val1": 1, "test1_val2": 1 }, step_index=1), TestFactory.create_workflow_step("tvb_test.adapters.testadapter3", "TestAdapter3", dynamic_kwargs={ "test": { wf_cfg.DATATYPE_INDEX_KEY: 0, wf_cfg.STEP_INDEX_KEY: 1 } }, step_index=2) ] self.__create_complex_workflow(workflow_step_list) stored_datatypes = dao.get_datatypes_info_for_project( self.test_project.id) self.assertTrue( len(stored_datatypes) == 3, "DataType from all step were not stored.") for result_row in stored_datatypes: self.assertTrue(result_row[0] in ['Datatype1', 'Datatype2'], "Wrong type was stored.") finished, started, error, _ = dao.get_operation_numbers( self.test_project.id) self.assertEqual( finished, 3, "Didn't start operations for both adapters in workflow.") self.assertEqual(started, 0, "Some operations from workflow didn't finish.") self.assertEqual(error, 0, "Some operations finished with error status.") def test_configuration2workflow(self): """ Test that building a WorflowStep from a WorkflowStepConfiguration. Make sure all the data is correctly passed. Also check that any base_wf_step is incremented to dynamic parameters step index. """ workflow_step = TestFactory.create_workflow_step( "tvb_test.adapters.testadapter1", "TestAdapter1", static_kwargs={"static_param": "test"}, dynamic_kwargs={ "dynamic_param": { wf_cfg.STEP_INDEX_KEY: 0, wf_cfg.DATATYPE_INDEX_KEY: 0 } }, step_index=1, base_step=5) self.assertEqual(workflow_step.step_index, 1, "Wrong step index in created workflow step.") self.assertEqual(workflow_step.static_param, {'static_param': 'test'}, 'Different static parameters on step.') self.assertEqual( workflow_step.dynamic_param, { 'dynamic_param': { wf_cfg.STEP_INDEX_KEY: 5, wf_cfg.DATATYPE_INDEX_KEY: 0 } }, "Dynamic parameters not saved properly, or base workflow index not added to step index." ) def test_create_workflow(self): """ Test that a workflow with all the associated workflow steps is actually created. """ workflow_step_list = [ TestFactory.create_workflow_step("tvb_test.adapters.testadapter2", "TestAdapter2", static_kwargs={"test2": 2}, step_index=1), TestFactory.create_workflow_step("tvb_test.adapters.testadapter1", "TestAdapter1", static_kwargs={ "test1_val1": 1, "test1_val2": 1 }, step_index=2) ] burst_id = self.__create_complex_workflow(workflow_step_list) workflow_entities = dao.get_workflows_for_burst(burst_id) self.assertTrue( len(workflow_entities) == 1, "For some reason workflow was not stored in database.") workflow_steps = dao.get_workflow_steps(workflow_entities[0].id) self.assertEqual(len(workflow_steps), len(workflow_step_list) + 1, "Wrong number of workflow steps created.")
class SpatioTemporalController(base.BaseController): """ Base class which contains methods related to spatio-temporal actions. """ def __init__(self): base.BaseController.__init__(self) self.flow_service = FlowService() self.logger = get_logger(__name__) editable_entities = [ dict(link='/spatial/stimulus/region/step_1_submit/1/1', title='Region Stimulus', subsection='regionstim', description='Create a new Stimulus on Region level'), dict(link='/spatial/stimulus/surface/step_1_submit/1/1', title='Surface Stimulus', subsection='surfacestim', description='Create a new Stimulus on Surface level') ] self.submenu_list = editable_entities @cherrypy.expose @using_template('base_template') @logged() @settings() def index(self, **data): """ Displays the main page for the spatio temporal section. """ template_specification = dict(title="Spatio temporal", data=data) template_specification['mainContent'] = 'header_menu' return self.fill_default_attributes(template_specification) @staticmethod def get_connectivity_parameters(input_connectivity, surface_data=None): """ Returns a dictionary which contains all the needed data for drawing a connectivity. """ viewer = ConnectivityViewer() global_params, global_pages = viewer.compute_connectivity_global_params( input_connectivity, surface_data) global_params.update(global_pages) global_params['selectedConnectivityGid'] = input_connectivity.gid return global_params def get_data_from_burst_configuration(self): """ Returns the model, integrator, connectivity and surface instances from the burst configuration. """ ### Read from session current burst-configuration burst_configuration = base.get_from_session(base.KEY_BURST_CONFIG) if burst_configuration is None: return None, None, None first_range = burst_configuration.get_simulation_parameter_value( 'first_range') second_range = burst_configuration.get_simulation_parameter_value( 'second_range') if ((first_range is not None and str(first_range).startswith(MODEL_PARAMETERS)) or (second_range is not None and str(second_range).startswith(MODEL_PARAMETERS))): base.set_error_message( "When configuring model parameters you are not allowed to specify range values." ) raise cherrypy.HTTPRedirect("/burst/") group = self.flow_service.get_algorithm_by_module_and_class( SIMULATOR_MODULE, SIMULATOR_CLASS)[1] simulator_adapter = self.flow_service.build_adapter_instance(group) try: params_dict = simulator_adapter.convert_ui_inputs( burst_configuration.get_all_simulator_values()[0], False) except Exception, excep: self.logger.exception(excep) base.set_error_message( "Some of the provided parameters have an invalid value.") raise cherrypy.HTTPRedirect("/burst/") ### Prepare Model instance model = burst_configuration.get_simulation_parameter_value(PARAM_MODEL) model_parameters = params_dict[MODEL_PARAMETERS] noise_framework.build_noise(model_parameters) try: model = get_traited_instance_for_name(model, Model, model_parameters) except Exception, ex: self.logger.exception(ex) self.logger.info( "Could not create the model instance with the given parameters. " "A new model instance will be created with the default values." ) model = get_traited_instance_for_name(model, Model, {})
class BaseController(object): """ This class contains the methods served at the root of the Web site. """ def __init__(self): self.logger = get_logger(self.__class__.__module__) self.version_info = None self.user_service = UserService() self.flow_service = FlowService() analyze_category = self.flow_service.get_launchable_non_viewers() self.analyze_category_link = '/flow/step/' + str(analyze_category.id) self.analyze_adapters = None self.connectivity_tab_link = '/flow/step_connectivity' view_category = self.flow_service.get_visualisers_category() conn_id = self.flow_service.get_algorithm_by_module_and_class( CONNECTIVITY_MODULE, CONNECTIVITY_CLASS)[1].id connectivity_link = self.get_url_adapter(view_category.id, conn_id) local_connectivity_link = '/spatial/localconnectivity/step_1/1' connectivity_submenu = [ dict(title="Large Scale Connectivity", subsection="connectivity", description= "View Connectivity Regions. Perform Connectivity lesions", link=connectivity_link), dict(title="Local Connectivity", subsection="local", link=local_connectivity_link, description= "Create or view existent Local Connectivity entities.") ] self.connectivity_submenu = connectivity_submenu @staticmethod def mark_file_for_delete(file_name, delete_parent_folder=False): """ This method stores provided file name in session, and later on when request is done, all these files/folders are deleted :param file_name: name of the file or folder to be deleted :param delete_parent_fodler: specify if the parent folder of the file should be removed too. """ # No processing if no file specified if file_name is None: return files_list = get_from_session(FILES_TO_DELETE_ATTR) if files_list is None: files_list = [] add2session(FILES_TO_DELETE_ATTR, files_list) # Now add file/folder to list if delete_parent_folder: folder_name = os.path.split(file_name)[0] files_list.append(folder_name) else: files_list.append(file_name) def _mark_selected(self, project): """ Set the project passed as parameter as the selected project. """ previous_project = get_current_project() ### Update project stored in selection, with latest Project entity from DB. members = self.user_service.get_users_for_project("", project.id)[1] project.members = members remove_from_session(KEY_CACHED_SIMULATOR_TREE) add2session(KEY_PROJECT, project) if previous_project is None or previous_project.id != project.id: ### Clean Burst selection from session in case of a different project. remove_from_session(KEY_BURST_CONFIG) ### Store in DB new project selection user = get_from_session(KEY_USER) if user is not None: self.user_service.save_project_to_user(user.id, project.id) ### Display info message about project change self.logger.debug("Selected project is now " + project.name) set_info_message("Your current working project is: " + str(project.name)) @staticmethod def get_url_adapter(step_key, adapter_id, back_page=None): """ Compute the URLs for a given adapter. Same URL is used both for GET and POST. """ result_url = '/flow/' + str(step_key) + '/' + str(adapter_id) if back_page is not None: result_url = result_url + "?back_page=" + str(back_page) return result_url @cherrypy.expose def index(self): """ / Path response Redirects to /tvb """ raise cherrypy.HTTPRedirect('/user') @cherrypy.expose() @using_template('user/base_user') def tvb(self, error=False, **data): """ /tvb URL Returns the home page with the messages stored in the user's session. """ self.logger.debug("Unused submit attributes:" + str(data)) template_dictionary = dict(mainContent="../index", title="The Virtual Brain Project") template_dictionary = self._fill_user_specific_attributes( template_dictionary) if get_from_session(KEY_IS_RESTART): template_dictionary[KEY_IS_RESTART] = True remove_from_session(KEY_IS_RESTART) return self.fill_default_attributes(template_dictionary, error) @cherrypy.expose @using_template('user/base_user') def error(self, **data): """Error page to redirect when something extremely bad happened""" template_specification = dict(mainContent="../error", title="Error page", data=data) template_specification = self._fill_user_specific_attributes( template_specification) return self.fill_default_attributes(template_specification) def _populate_user_and_project(self, template_dictionary, escape_db_operations=False): """ Populate the template dictionary with current logged user (from session). """ logged_user = get_logged_user() template_dictionary[KEY_USER] = logged_user show_help = logged_user is not None and logged_user.is_online_help_active( ) template_dictionary[KEY_SHOW_ONLINE_HELP] = show_help project = get_current_project() template_dictionary[KEY_PROJECT] = project if project is not None and not escape_db_operations: self.update_operations_count() return template_dictionary @staticmethod def _populate_message(template_dictionary): """ Populate the template dictionary with current message stored in session. Also specify the message type (default INFO). Clear from session current message (to avoid displaying it twice). """ message_type = remove_from_session(KEY_MESSAGE_TYPE) if message_type is None: message_type = TYPE_INFO template_dictionary[KEY_MESSAGE_TYPE] = message_type message = remove_from_session(KEY_MESSAGE) if message is None: message = "" template_dictionary[KEY_MESSAGE] = message return template_dictionary def _populate_menu(self, template_dictionary): """ Populate current template with information for the Left Menu. """ if KEY_FIRST_RUN not in template_dictionary: template_dictionary[KEY_FIRST_RUN] = False template_dictionary[KEY_LINK_ANALYZE] = self.analyze_category_link template_dictionary[ KEY_LINK_CONNECTIVITY_TAB] = self.connectivity_tab_link if KEY_BACK_PAGE not in template_dictionary: template_dictionary[KEY_BACK_PAGE] = False template_dictionary[ KEY_SECTION_TITLES] = WebStructure.WEB_SECTION_TITLES template_dictionary[ KEY_SUBSECTION_TITLES] = WebStructure.WEB_SUBSECTION_TITLES return template_dictionary def _populate_section(self, algo_group, result_template): """ Populate Section and Sub-Section fields from current Algorithm-Group. """ if algo_group.module == CONNECTIVITY_MODULE: result_template[KEY_SECTION] = 'connectivity' result_template[KEY_SUB_SECTION] = 'connectivity' result_template[KEY_SUBMENU_LIST] = self.connectivity_submenu elif algo_group.group_category.display: ### Visualizers on the Burst Page result_template[KEY_SECTION] = 'burst' result_template[ KEY_SUB_SECTION] = 'view_' + algo_group.subsection_name elif algo_group.group_category.rawinput: ### Upload algorithms result_template[KEY_SECTION] = 'project' result_template[KEY_SUB_SECTION] = 'data' elif 'RAW_DATA' in algo_group.group_category.defaultdatastate: ### Creators result_template[KEY_SECTION] = 'stimulus' result_template[KEY_SUB_SECTION] = 'stimulus' else: ### Analyzers result_template[ KEY_SECTION] = algo_group.group_category.displayname.lower() result_template[KEY_SUB_SECTION] = algo_group.subsection_name result_template[KEY_SUBMENU_LIST] = self.analyze_adapters def _fill_user_specific_attributes(self, template_dictionary): """ Attributes needed for base_user template. """ template_dictionary[KEY_INCLUDE_TOOLTIP] = False template_dictionary[KEY_WRAP_CONTENT_IN_MAIN_DIV] = True template_dictionary[KEY_CURRENT_TAB] = 'none' return template_dictionary def fill_default_attributes(self, template_dictionary, escape_db_operations=False): """ Fill into 'template_dictionary' data that we want to have ready in UI. """ template_dictionary = self._populate_user_and_project( template_dictionary, escape_db_operations) template_dictionary = self._populate_message(template_dictionary) template_dictionary = self._populate_menu(template_dictionary) if KEY_ERRORS not in template_dictionary: template_dictionary[KEY_ERRORS] = {} if KEY_FORM_DATA not in template_dictionary: template_dictionary[KEY_FORM_DATA] = {} if KEY_SUB_SECTION not in template_dictionary and KEY_SECTION in template_dictionary: template_dictionary[KEY_SUB_SECTION] = template_dictionary[ KEY_SECTION] if KEY_SUBMENU_LIST not in template_dictionary: template_dictionary[KEY_SUBMENU_LIST] = None template_dictionary[KEY_CURRENT_VERSION] = cfg.BASE_VERSION return template_dictionary def fill_overlay_attributes(self, template_dictionary, title, description, content_template, css_class, tabs=None, overlay_indexes=None): """ This method prepares parameters for rendering overlay (overlay.html) :param title: overlay title :param description: overlay description :param content_template: path&name of the template file which will fill overlay content (without .html) :param css_class: CSS class to be applied on overlay :param tabs: list of strings containing names of the tabs """ if template_dictionary is None: template_dictionary = dict() template_dictionary[KEY_OVERLAY_TITLE] = title template_dictionary[KEY_OVERLAY_DESCRIPTION] = description template_dictionary[KEY_OVERLAY_CONTENT_TEMPLATE] = content_template template_dictionary[KEY_OVERLAY_CLASS] = css_class template_dictionary[ KEY_OVERLAY_TABS] = tabs if tabs is not None and len( tabs) > 0 else [] if overlay_indexes is not None: template_dictionary[KEY_OVERLAY_INDEXES] = overlay_indexes else: template_dictionary[KEY_OVERLAY_INDEXES] = range( len(tabs)) if tabs is not None else [] template_dictionary[KEY_OVERLAY_PAGINATION] = False return template_dictionary @cherrypy.expose @using_template('overlay_blocker') def showBlockerOverlay(self, **data): """ Returns the content of the blocking overlay (covers entire page and do not allow any action) """ return self.fill_default_attributes(dict(data)) def update_operations_count(self): """ If a project is selected, update Operation Numbers in call-out. """ project = get_current_project() if project is not None: fns, sta, err, canceled = self.flow_service.get_operation_numbers( project.id) project.operations_finished = fns project.operations_started = sta project.operations_error = err project.operations_canceled = canceled add2session(KEY_PROJECT, project)