class WorkflowTest(TransactionalTestCase):
"""
Test that workflow conversion methods are valid.
"""
def setUp(self):
# self.clean_database()
self.test_user = TestFactory.create_user()
self.test_project = TestFactory.create_project(self.test_user)
self.old_config_file = cfg.CURRENT_DIR
cfg.CURRENT_DIR = os.path.dirname(tvb_test.__file__)
self.workflow_service = WorkflowService()
self.burst_service = BurstService()
self.operation_service = OperationService()
self.flow_service = FlowService()
def tearDown(self):
"""
Remove project folders and clean up database.
"""
FilesHelper().remove_project_structure(self.test_project.name)
self.delete_project_folders()
cfg.CURRENT_DIR = self.old_config_file
def __create_complex_workflow(self, workflow_step_list):
"""
Creates a burst with a complex workflow with a given list of workflow steps.
@param workflow_step_list: a lsit of workflow steps that will be used in the
creation of a new workflow for a new burst
"""
burst_config = TestFactory.store_burst(self.test_project.id)
stored_dt = datatypes_factory.DatatypesFactory()._store_datatype(
Datatype1())
first_step_algorithm = self.flow_service.get_algorithm_by_module_and_class(
"tvb_test.adapters.testadapter1", "TestAdapterDatatypeInput")[0]
metadata = {DataTypeMetaData.KEY_BURST: burst_config.id}
kwargs = {"test_dt_input": stored_dt.gid, 'test_non_dt_input': '0'}
operations, group = self.operation_service.prepare_operations(
self.test_user.id, self.test_project.id, first_step_algorithm,
first_step_algorithm.algo_group.group_category, metadata, **kwargs)
workflows = self.workflow_service.create_and_store_workflow(
project_id=self.test_project.id,
burst_id=burst_config.id,
simulator_index=0,
simulator_id=first_step_algorithm.id,
operations=operations)
self.operation_service.prepare_operations_for_workflowsteps(
workflow_step_list, workflows, self.test_user.id, burst_config.id,
self.test_project.id, group, operations)
#fire the first op
if len(operations) > 0:
self.operation_service.launch_operation(operations[0].id, False)
return burst_config.id
def test_workflow_generation(self):
"""
A simple test just for the fact that a workflow is created an ran,
no dynamic parameters are passed. In this case we create a two steps
workflow: step1 - tvb_test.adapters.testadapter2.TestAdapter2
step2 - tvb_test.adapters.testadapter1.TestAdapter1
The first adapter doesn't return anything and the second returns one
tvb.datatypes.datatype1.Datatype1 instance. We check that the steps
are actually ran by checking that two operations are created and that
one dataType is stored.
"""
workflow_step_list = [
TestFactory.create_workflow_step("tvb_test.adapters.testadapter2",
"TestAdapter2",
static_kwargs={"test2": 2},
step_index=1),
TestFactory.create_workflow_step("tvb_test.adapters.testadapter1",
"TestAdapter1",
static_kwargs={
"test1_val1": 1,
"test1_val2": 1
},
step_index=2)
]
self.__create_complex_workflow(workflow_step_list)
stored_datatypes = dao.get_datatypes_info_for_project(
self.test_project.id)
self.assertTrue(
len(stored_datatypes) == 2,
"DataType from second step was not stored.")
self.assertTrue(stored_datatypes[0][0] == 'Datatype1',
"Wrong type was stored.")
self.assertTrue(stored_datatypes[1][0] == 'Datatype1',
"Wrong type was stored.")
finished, started, error, _ = dao.get_operation_numbers(
self.test_project.id)
self.assertEqual(
finished, 3,
"Didnt start operations for both adapters in workflow.")
self.assertEqual(started, 0,
"Some operations from workflow didnt finish.")
self.assertEqual(error, 0,
"Some operations finished with error status.")
def test_workflow_dynamic_params(self):
"""
A simple test just for the fact that dynamic parameters are passed properly
between two workflow steps:
step1 - tvb_test.adapters.testadapter1.TestAdapter1
step2 - tvb_test.adapters.testadapter3.TestAdapter3
The first adapter returns a tvb.datatypes.datatype1.Datatype1 instance.
The second adapter has this passed as a dynamic workflow parameter.
We check that the steps are actually ran by checking that two operations
are created and that two dataTypes are stored.
"""
workflow_step_list = [
TestFactory.create_workflow_step("tvb_test.adapters.testadapter1",
"TestAdapter1",
static_kwargs={
"test1_val1": 1,
"test1_val2": 1
},
step_index=1),
TestFactory.create_workflow_step("tvb_test.adapters.testadapter3",
"TestAdapter3",
dynamic_kwargs={
"test": {
wf_cfg.DATATYPE_INDEX_KEY:
0,
wf_cfg.STEP_INDEX_KEY: 1
}
},
step_index=2)
]
self.__create_complex_workflow(workflow_step_list)
stored_datatypes = dao.get_datatypes_info_for_project(
self.test_project.id)
self.assertTrue(
len(stored_datatypes) == 3,
"DataType from all step were not stored.")
for result_row in stored_datatypes:
self.assertTrue(result_row[0] in ['Datatype1', 'Datatype2'],
"Wrong type was stored.")
finished, started, error, _ = dao.get_operation_numbers(
self.test_project.id)
self.assertEqual(
finished, 3,
"Didn't start operations for both adapters in workflow.")
self.assertEqual(started, 0,
"Some operations from workflow didn't finish.")
self.assertEqual(error, 0,
"Some operations finished with error status.")
def test_configuration2workflow(self):
"""
Test that building a WorflowStep from a WorkflowStepConfiguration. Make sure all the data is
correctly passed. Also check that any base_wf_step is incremented to dynamic parameters step index.
"""
workflow_step = TestFactory.create_workflow_step(
"tvb_test.adapters.testadapter1",
"TestAdapter1",
static_kwargs={"static_param": "test"},
dynamic_kwargs={
"dynamic_param": {
wf_cfg.STEP_INDEX_KEY: 0,
wf_cfg.DATATYPE_INDEX_KEY: 0
}
},
step_index=1,
base_step=5)
self.assertEqual(workflow_step.step_index, 1,
"Wrong step index in created workflow step.")
self.assertEqual(workflow_step.static_param, {'static_param': 'test'},
'Different static parameters on step.')
self.assertEqual(
workflow_step.dynamic_param, {
'dynamic_param': {
wf_cfg.STEP_INDEX_KEY: 5,
wf_cfg.DATATYPE_INDEX_KEY: 0
}
},
"Dynamic parameters not saved properly, or base workflow index not added to step index."
)
def test_create_workflow(self):
"""
Test that a workflow with all the associated workflow steps is actually created.
"""
workflow_step_list = [
TestFactory.create_workflow_step("tvb_test.adapters.testadapter2",
"TestAdapter2",
static_kwargs={"test2": 2},
step_index=1),
TestFactory.create_workflow_step("tvb_test.adapters.testadapter1",
"TestAdapter1",
static_kwargs={
"test1_val1": 1,
"test1_val2": 1
},
step_index=2)
]
burst_id = self.__create_complex_workflow(workflow_step_list)
workflow_entities = dao.get_workflows_for_burst(burst_id)
self.assertTrue(
len(workflow_entities) == 1,
"For some reason workflow was not stored in database.")
workflow_steps = dao.get_workflow_steps(workflow_entities[0].id)
self.assertEqual(len(workflow_steps),
len(workflow_step_list) + 1,
"Wrong number of workflow steps created.")
class SpatioTemporalController(base.BaseController):
"""
Base class which contains methods related to spatio-temporal actions.
"""
def __init__(self):
base.BaseController.__init__(self)
self.flow_service = FlowService()
self.logger = get_logger(__name__)
editable_entities = [
dict(link='/spatial/stimulus/region/step_1_submit/1/1',
title='Region Stimulus',
subsection='regionstim',
description='Create a new Stimulus on Region level'),
dict(link='/spatial/stimulus/surface/step_1_submit/1/1',
title='Surface Stimulus',
subsection='surfacestim',
description='Create a new Stimulus on Surface level')
]
self.submenu_list = editable_entities
@cherrypy.expose
@using_template('base_template')
@logged()
@settings()
def index(self, **data):
"""
Displays the main page for the spatio temporal section.
"""
template_specification = dict(title="Spatio temporal", data=data)
template_specification['mainContent'] = 'header_menu'
return self.fill_default_attributes(template_specification)
@staticmethod
def get_connectivity_parameters(input_connectivity, surface_data=None):
"""
Returns a dictionary which contains all the needed data for drawing a connectivity.
"""
viewer = ConnectivityViewer()
global_params, global_pages = viewer.compute_connectivity_global_params(
input_connectivity, surface_data)
global_params.update(global_pages)
global_params['selectedConnectivityGid'] = input_connectivity.gid
return global_params
def get_data_from_burst_configuration(self):
"""
Returns the model, integrator, connectivity and surface instances from the burst configuration.
"""
### Read from session current burst-configuration
burst_configuration = base.get_from_session(base.KEY_BURST_CONFIG)
if burst_configuration is None:
return None, None, None
first_range = burst_configuration.get_simulation_parameter_value(
'first_range')
second_range = burst_configuration.get_simulation_parameter_value(
'second_range')
if ((first_range is not None
and str(first_range).startswith(MODEL_PARAMETERS))
or (second_range is not None
and str(second_range).startswith(MODEL_PARAMETERS))):
base.set_error_message(
"When configuring model parameters you are not allowed to specify range values."
)
raise cherrypy.HTTPRedirect("/burst/")
group = self.flow_service.get_algorithm_by_module_and_class(
SIMULATOR_MODULE, SIMULATOR_CLASS)[1]
simulator_adapter = self.flow_service.build_adapter_instance(group)
try:
params_dict = simulator_adapter.convert_ui_inputs(
burst_configuration.get_all_simulator_values()[0], False)
except Exception, excep:
self.logger.exception(excep)
base.set_error_message(
"Some of the provided parameters have an invalid value.")
raise cherrypy.HTTPRedirect("/burst/")
### Prepare Model instance
model = burst_configuration.get_simulation_parameter_value(PARAM_MODEL)
model_parameters = params_dict[MODEL_PARAMETERS]
noise_framework.build_noise(model_parameters)
try:
model = get_traited_instance_for_name(model, Model,
model_parameters)
except Exception, ex:
self.logger.exception(ex)
self.logger.info(
"Could not create the model instance with the given parameters. "
"A new model instance will be created with the default values."
)
model = get_traited_instance_for_name(model, Model, {})
class BaseController(object):
"""
This class contains the methods served at the root of the Web site.
"""
def __init__(self):
self.logger = get_logger(self.__class__.__module__)
self.version_info = None
self.user_service = UserService()
self.flow_service = FlowService()
analyze_category = self.flow_service.get_launchable_non_viewers()
self.analyze_category_link = '/flow/step/' + str(analyze_category.id)
self.analyze_adapters = None
self.connectivity_tab_link = '/flow/step_connectivity'
view_category = self.flow_service.get_visualisers_category()
conn_id = self.flow_service.get_algorithm_by_module_and_class(
CONNECTIVITY_MODULE, CONNECTIVITY_CLASS)[1].id
connectivity_link = self.get_url_adapter(view_category.id, conn_id)
local_connectivity_link = '/spatial/localconnectivity/step_1/1'
connectivity_submenu = [
dict(title="Large Scale Connectivity",
subsection="connectivity",
description=
"View Connectivity Regions. Perform Connectivity lesions",
link=connectivity_link),
dict(title="Local Connectivity",
subsection="local",
link=local_connectivity_link,
description=
"Create or view existent Local Connectivity entities.")
]
self.connectivity_submenu = connectivity_submenu
@staticmethod
def mark_file_for_delete(file_name, delete_parent_folder=False):
"""
This method stores provided file name in session,
and later on when request is done, all these files/folders
are deleted
:param file_name: name of the file or folder to be deleted
:param delete_parent_fodler: specify if the parent folder of the
file should be removed too.
"""
# No processing if no file specified
if file_name is None:
return
files_list = get_from_session(FILES_TO_DELETE_ATTR)
if files_list is None:
files_list = []
add2session(FILES_TO_DELETE_ATTR, files_list)
# Now add file/folder to list
if delete_parent_folder:
folder_name = os.path.split(file_name)[0]
files_list.append(folder_name)
else:
files_list.append(file_name)
def _mark_selected(self, project):
"""
Set the project passed as parameter as the selected project.
"""
previous_project = get_current_project()
### Update project stored in selection, with latest Project entity from DB.
members = self.user_service.get_users_for_project("", project.id)[1]
project.members = members
remove_from_session(KEY_CACHED_SIMULATOR_TREE)
add2session(KEY_PROJECT, project)
if previous_project is None or previous_project.id != project.id:
### Clean Burst selection from session in case of a different project.
remove_from_session(KEY_BURST_CONFIG)
### Store in DB new project selection
user = get_from_session(KEY_USER)
if user is not None:
self.user_service.save_project_to_user(user.id, project.id)
### Display info message about project change
self.logger.debug("Selected project is now " + project.name)
set_info_message("Your current working project is: " +
str(project.name))
@staticmethod
def get_url_adapter(step_key, adapter_id, back_page=None):
"""
Compute the URLs for a given adapter.
Same URL is used both for GET and POST.
"""
result_url = '/flow/' + str(step_key) + '/' + str(adapter_id)
if back_page is not None:
result_url = result_url + "?back_page=" + str(back_page)
return result_url
@cherrypy.expose
def index(self):
"""
/ Path response
Redirects to /tvb
"""
raise cherrypy.HTTPRedirect('/user')
@cherrypy.expose()
@using_template('user/base_user')
def tvb(self, error=False, **data):
"""
/tvb URL
Returns the home page with the messages stored in the user's session.
"""
self.logger.debug("Unused submit attributes:" + str(data))
template_dictionary = dict(mainContent="../index",
title="The Virtual Brain Project")
template_dictionary = self._fill_user_specific_attributes(
template_dictionary)
if get_from_session(KEY_IS_RESTART):
template_dictionary[KEY_IS_RESTART] = True
remove_from_session(KEY_IS_RESTART)
return self.fill_default_attributes(template_dictionary, error)
@cherrypy.expose
@using_template('user/base_user')
def error(self, **data):
"""Error page to redirect when something extremely bad happened"""
template_specification = dict(mainContent="../error",
title="Error page",
data=data)
template_specification = self._fill_user_specific_attributes(
template_specification)
return self.fill_default_attributes(template_specification)
def _populate_user_and_project(self,
template_dictionary,
escape_db_operations=False):
"""
Populate the template dictionary with current logged user (from session).
"""
logged_user = get_logged_user()
template_dictionary[KEY_USER] = logged_user
show_help = logged_user is not None and logged_user.is_online_help_active(
)
template_dictionary[KEY_SHOW_ONLINE_HELP] = show_help
project = get_current_project()
template_dictionary[KEY_PROJECT] = project
if project is not None and not escape_db_operations:
self.update_operations_count()
return template_dictionary
@staticmethod
def _populate_message(template_dictionary):
"""
Populate the template dictionary with current message stored in session.
Also specify the message type (default INFO).
Clear from session current message (to avoid displaying it twice).
"""
message_type = remove_from_session(KEY_MESSAGE_TYPE)
if message_type is None:
message_type = TYPE_INFO
template_dictionary[KEY_MESSAGE_TYPE] = message_type
message = remove_from_session(KEY_MESSAGE)
if message is None:
message = ""
template_dictionary[KEY_MESSAGE] = message
return template_dictionary
def _populate_menu(self, template_dictionary):
"""
Populate current template with information for the Left Menu.
"""
if KEY_FIRST_RUN not in template_dictionary:
template_dictionary[KEY_FIRST_RUN] = False
template_dictionary[KEY_LINK_ANALYZE] = self.analyze_category_link
template_dictionary[
KEY_LINK_CONNECTIVITY_TAB] = self.connectivity_tab_link
if KEY_BACK_PAGE not in template_dictionary:
template_dictionary[KEY_BACK_PAGE] = False
template_dictionary[
KEY_SECTION_TITLES] = WebStructure.WEB_SECTION_TITLES
template_dictionary[
KEY_SUBSECTION_TITLES] = WebStructure.WEB_SUBSECTION_TITLES
return template_dictionary
def _populate_section(self, algo_group, result_template):
"""
Populate Section and Sub-Section fields from current Algorithm-Group.
"""
if algo_group.module == CONNECTIVITY_MODULE:
result_template[KEY_SECTION] = 'connectivity'
result_template[KEY_SUB_SECTION] = 'connectivity'
result_template[KEY_SUBMENU_LIST] = self.connectivity_submenu
elif algo_group.group_category.display:
### Visualizers on the Burst Page
result_template[KEY_SECTION] = 'burst'
result_template[
KEY_SUB_SECTION] = 'view_' + algo_group.subsection_name
elif algo_group.group_category.rawinput:
### Upload algorithms
result_template[KEY_SECTION] = 'project'
result_template[KEY_SUB_SECTION] = 'data'
elif 'RAW_DATA' in algo_group.group_category.defaultdatastate:
### Creators
result_template[KEY_SECTION] = 'stimulus'
result_template[KEY_SUB_SECTION] = 'stimulus'
else:
### Analyzers
result_template[
KEY_SECTION] = algo_group.group_category.displayname.lower()
result_template[KEY_SUB_SECTION] = algo_group.subsection_name
result_template[KEY_SUBMENU_LIST] = self.analyze_adapters
def _fill_user_specific_attributes(self, template_dictionary):
"""
Attributes needed for base_user template.
"""
template_dictionary[KEY_INCLUDE_TOOLTIP] = False
template_dictionary[KEY_WRAP_CONTENT_IN_MAIN_DIV] = True
template_dictionary[KEY_CURRENT_TAB] = 'none'
return template_dictionary
def fill_default_attributes(self,
template_dictionary,
escape_db_operations=False):
"""
Fill into 'template_dictionary' data that we want to have ready in UI.
"""
template_dictionary = self._populate_user_and_project(
template_dictionary, escape_db_operations)
template_dictionary = self._populate_message(template_dictionary)
template_dictionary = self._populate_menu(template_dictionary)
if KEY_ERRORS not in template_dictionary:
template_dictionary[KEY_ERRORS] = {}
if KEY_FORM_DATA not in template_dictionary:
template_dictionary[KEY_FORM_DATA] = {}
if KEY_SUB_SECTION not in template_dictionary and KEY_SECTION in template_dictionary:
template_dictionary[KEY_SUB_SECTION] = template_dictionary[
KEY_SECTION]
if KEY_SUBMENU_LIST not in template_dictionary:
template_dictionary[KEY_SUBMENU_LIST] = None
template_dictionary[KEY_CURRENT_VERSION] = cfg.BASE_VERSION
return template_dictionary
def fill_overlay_attributes(self,
template_dictionary,
title,
description,
content_template,
css_class,
tabs=None,
overlay_indexes=None):
"""
This method prepares parameters for rendering overlay (overlay.html)
:param title: overlay title
:param description: overlay description
:param content_template: path&name of the template file which will fill overlay content (without .html)
:param css_class: CSS class to be applied on overlay
:param tabs: list of strings containing names of the tabs
"""
if template_dictionary is None:
template_dictionary = dict()
template_dictionary[KEY_OVERLAY_TITLE] = title
template_dictionary[KEY_OVERLAY_DESCRIPTION] = description
template_dictionary[KEY_OVERLAY_CONTENT_TEMPLATE] = content_template
template_dictionary[KEY_OVERLAY_CLASS] = css_class
template_dictionary[
KEY_OVERLAY_TABS] = tabs if tabs is not None and len(
tabs) > 0 else []
if overlay_indexes is not None:
template_dictionary[KEY_OVERLAY_INDEXES] = overlay_indexes
else:
template_dictionary[KEY_OVERLAY_INDEXES] = range(
len(tabs)) if tabs is not None else []
template_dictionary[KEY_OVERLAY_PAGINATION] = False
return template_dictionary
@cherrypy.expose
@using_template('overlay_blocker')
def showBlockerOverlay(self, **data):
"""
Returns the content of the blocking overlay (covers entire page and do not allow any action)
"""
return self.fill_default_attributes(dict(data))
def update_operations_count(self):
"""
If a project is selected, update Operation Numbers in call-out.
"""
project = get_current_project()
if project is not None:
fns, sta, err, canceled = self.flow_service.get_operation_numbers(
project.id)
project.operations_finished = fns
project.operations_started = sta
project.operations_error = err
project.operations_canceled = canceled
add2session(KEY_PROJECT, project)
