def test_new_portlet_configuration(self):
"""
Test that the correct portlet configuration is generated for the test portlet.
"""
# Passing an invalid portlet ID should fail and raise an InvalidPortletConfiguration exception.
self.assertRaises(InvalidPortletConfiguration,
self.burst_service.new_portlet_configuration, -1)
# Now the happy flow
test_portlet = dao.get_portlet_by_identifier(self.PORTLET_ID)
portlet_configuration = self.burst_service.new_portlet_configuration(
test_portlet.id)
analyzers = portlet_configuration.analyzers
self.assertEqual(
len(analyzers), 1, "Portlet configuration not build properly."
" Portlet's analyzers list has unexpected number of elements.")
self.assertEqual(
analyzers[0].dynamic_param, {
u'test_dt_input': {
wf_cfg.DATATYPE_INDEX_KEY: 0,
wf_cfg.STEP_INDEX_KEY: 0
}
}, "Dynamic parameters not loaded properly")
visualizer = portlet_configuration.visualizer
self.assertEqual(visualizer.dynamic_param, {},
"Dynamic parameters not loaded properly")
self.assertEqual(visualizer.static_param, {u'test2': u'0'},
'Static parameters not loaded properly')
def test_launch_group_burst_no_metric(self):
"""
Test the launch burst method from burst service. Try to launch a burst with test adapter which has
no metrics associated. This should fail.
"""
burst_config = self.burst_service.new_burst_configuration(
self.test_project.id)
algo_id = self.flow_service.get_algorithm_by_module_and_class(
'tvb.tests.framework.adapters.testadapter1', 'TestAdapter1').id
kwargs_replica = {
'test1_val1': '[0, 1, 2]',
'test1_val2': '0',
RANGE_PARAMETER_1: 'test1_val1'
}
test_portlet = dao.get_portlet_by_identifier(self.PORTLET_ID)
tab_config = {test_portlet.id: [(0, 0), (0, 1), (1, 0)]}
self._add_portlets_to_burst(burst_config, tab_config)
burst_config.update_simulator_configuration(kwargs_replica)
burst_id, _ = self.burst_service.launch_burst(burst_config, 0, algo_id,
self.test_user.id)
burst_config = dao.get_burst_by_id(burst_id)
# Wait maximum x seconds for burst to finish
self._wait_for_burst(burst_config, error_expected=True)
launched_workflows = dao.get_workflows_for_burst(burst_id,
is_count=True)
assert 3 == launched_workflows, "3 workflows should have been launched due to group parameter."
op_groups = self.count_all_entities(OperationGroup)
dt_groups = self.count_all_entities(DataTypeGroup)
assert 5 == op_groups, "An operation group should have been created for each step."
assert 5 == dt_groups, "An dataType group should have been created for each step."
def test_load_tab_configuration(self):
"""
Create a burst with some predefined portlets in some known positions. Check that the
load_tab_configuration method does what it is expected, and we get the portlets in the
corresponding tab positions.
"""
burst_config = self.burst_service.new_burst_configuration(self.test_project.id)
SIMULATOR_MODULE = 'tvb.tests.framework.adapters.testadapter1'
SIMULATOR_CLASS = 'TestAdapter1'
algo_id = self.flow_service.get_algorithm_by_module_and_class(SIMULATOR_MODULE, SIMULATOR_CLASS).id
kwargs_replica = {'test1_val1': '0', 'test1_val2': '0'}
test_portlet = dao.get_portlet_by_identifier(self.PORTLET_ID)
# Add test_portlet to positions (0,0), (0,1) and (1,0)
tab_config = {test_portlet.id: [(0, 0), (0, 1), (1, 0)]}
self._add_portlets_to_burst(burst_config, tab_config)
burst_config.update_simulator_configuration(kwargs_replica)
burst_id, _ = self.burst_service.launch_burst(burst_config, 0, algo_id, self.test_user.id)
burst_config = dao.get_burst_by_id(burst_id)
burst_config = self._wait_for_burst(burst_config)
burst_wf = dao.get_workflows_for_burst(burst_config.id)[0]
wf_step = dao.get_workflow_steps(burst_wf.id)[0]
burst_config.prepare_after_load()
for tab in burst_config.tabs:
for portlet in tab.portlets:
self.assertTrue(portlet is None, "Before loading the tab configuration all portlets should be none.")
burst_config = self.burst_service.load_tab_configuration(burst_config, wf_step.fk_operation)
for tab_idx, tab in enumerate(burst_config.tabs):
for portlet_idx, portlet in enumerate(tab.portlets):
if (tab_idx == 0 and portlet_idx in [0, 1]) or (tab_idx == 1 and portlet_idx == 0):
self.assertTrue(portlet is not None, "portlet gonfiguration not set")
self.assertEqual(test_portlet.id, portlet.portlet_id, "Unexpected portlet entity loaded.")
else:
self.assertTrue(portlet is None, "Before loading the tab configuration all portlets should be none")
def test_update_portlet_config(self):
"""
Test if a portlet configuration parameters are updated accordingly with a set
of overwrites that would normally come from UI. Make sure to restart only if
analyzer parameters change.
"""
def __update_params(declared_overwrites, expected_result):
"""
Do the update and check that we get indeed the expected_result.
:param declared_overwrites: a input dictionary in the form {'$$name$$' : '$$value$$'}. Make
sure $$name$$ has the prefix that is added in case of portlet parameters,
namely ADAPTER_PREFIX_ROOT + step_index + actual_name
:param expected_result: boolean which should represent if we need or not to restart. (Was a
visualizer parameter change or an analyzer one)
"""
result = self.burst_service.update_portlet_configuration(
portlet_configuration, declared_overwrites)
self.assertEqual(
expected_result, result,
"After update expected %s as 'need_restart' but got %s." %
(expected_result, result))
test_portlet = dao.get_portlet_by_identifier(self.PORTLET_ID)
portlet_configuration = self.burst_service.new_portlet_configuration(
test_portlet.id)
previous_entry = portlet_configuration.analyzers[0].static_param[
'test_non_dt_input']
declared_overwrites = {
ADAPTER_PREFIX_ROOT + '0test_non_dt_input': previous_entry
}
__update_params(declared_overwrites, False)
declared_overwrites = {ADAPTER_PREFIX_ROOT + '1test2': 'new_value'}
__update_params(declared_overwrites, False)
declared_overwrites = {ADAPTER_PREFIX_ROOT + '0test_non_dt_input': '1'}
__update_params(declared_overwrites, True)
def test_index(self):
"""
Test that index returns a dict with all required keys. Also check
that the default portlets are populated, with only the first being
the TimeSeries portlet and the rest are empty.
"""
result_dict = self.burst_c.index()
self.assertTrue('burst_list' in result_dict
and result_dict['burst_list'] == [])
self.assertTrue('available_metrics' in result_dict
and isinstance(result_dict['available_metrics'], list))
self.assertTrue('portletList' in result_dict
and isinstance(result_dict['portletList'], list))
self.assertEqual(result_dict[common.KEY_SECTION], "burst")
self.assertTrue(
'burstConfig' in result_dict
and isinstance(result_dict['burstConfig'], BurstConfiguration))
portlets = json.loads(result_dict['selectedPortlets'])
portlet_id = dao.get_portlet_by_identifier("TimeSeries").id
for tab_idx, tab in enumerate(portlets):
for index_in_tab, value in enumerate(tab):
if tab_idx == 0 and index_in_tab == 0:
self.assertEqual(value, [portlet_id, "TimeSeries"])
else:
self.assertEqual(value, [-1, "None"])
self.assertTrue(result_dict['draw_hidden_ranges'])
def test_update_portlet_config_invalid_data(self):
"""
Trying an update on a portlet configuration with invalid data
should not change the configuration instance in any way.
"""
test_portlet = dao.get_portlet_by_identifier(self.PORTLET_ID)
portlet_configuration = self.burst_service.new_portlet_configuration(
test_portlet.id)
invalid_overwrites = {
'this_is_not_a_valid_key': 'for_test_portlet_update'
}
before_update = copy.deepcopy(portlet_configuration)
self.burst_service.update_portlet_configuration(
portlet_configuration, invalid_overwrites)
self.assertEqual(set(dir(before_update)),
set(dir(portlet_configuration)))
#An update with invalid input data should have no effect on the configuration, but attributes changed
for key in portlet_configuration.__dict__.keys():
if hasattr(getattr(portlet_configuration, key), '__call__'):
self.assertEqual(
getattr(before_update, key),
getattr(portlet_configuration, key),
"The value of attribute %s changed by a update with invalid data "
"when it shouldn't have." % key)
def test_launch_burst(self):
"""
Test the launch burst method from burst service.
"""
first_step_algo = self.flow_service.get_algorithm_by_module_and_class(
'tvb.tests.framework.adapters.testadapter1', 'TestAdapter1')
adapter_interface = self.flow_service.prepare_adapter(self.test_project.id, first_step_algo)
ui_submited_simulator_iface_replica = {}
kwargs_replica = {}
for entry in adapter_interface:
ui_submited_simulator_iface_replica[entry[ABCAdapter.KEY_NAME]] = {model.KEY_PARAMETER_CHECKED: True,
model.KEY_SAVED_VALUE: entry[
ABCAdapter.KEY_DEFAULT]}
kwargs_replica[entry[ABCAdapter.KEY_NAME]] = entry[ABCAdapter.KEY_DEFAULT]
burst_config = self.burst_service.new_burst_configuration(self.test_project.id)
burst_config.simulator_configuration = ui_submited_simulator_iface_replica
test_portlet = dao.get_portlet_by_identifier(self.PORTLET_ID)
tab_config = {test_portlet.id: [(0, 0), (0, 1), (1, 0)]}
self._add_portlets_to_burst(burst_config, tab_config)
burst_config.update_simulator_configuration(kwargs_replica)
burst_id, _ = self.burst_service.launch_burst(burst_config, 0, first_step_algo.id, self.test_user.id)
burst_config = dao.get_burst_by_id(burst_id)
self.assertTrue(burst_config.status in (BurstConfiguration.BURST_FINISHED, BurstConfiguration.BURST_RUNNING),
"Burst not launched successfully!")
# Wait maximum x seconds for burst to finish
self._wait_for_burst(burst_config)
def test_launch_burst(self):
"""
Test the launch burst method from burst service.
"""
first_step_algo = self.flow_service.get_algorithm_by_module_and_class(
'tvb.tests.framework.adapters.testadapter1', 'TestAdapter1')
adapter_interface = self.flow_service.prepare_adapter(
self.test_project.id, first_step_algo)
ui_submited_simulator_iface_replica = {}
kwargs_replica = {}
for entry in adapter_interface:
ui_submited_simulator_iface_replica[entry[ABCAdapter.KEY_NAME]] = {
model.KEY_PARAMETER_CHECKED: True,
model.KEY_SAVED_VALUE: entry[ABCAdapter.KEY_DEFAULT]
}
kwargs_replica[entry[ABCAdapter.KEY_NAME]] = entry[
ABCAdapter.KEY_DEFAULT]
burst_config = self.burst_service.new_burst_configuration(
self.test_project.id)
burst_config.simulator_configuration = ui_submited_simulator_iface_replica
test_portlet = dao.get_portlet_by_identifier(self.PORTLET_ID)
tab_config = {test_portlet.id: [(0, 0), (0, 1), (1, 0)]}
self._add_portlets_to_burst(burst_config, tab_config)
burst_config.update_simulator_configuration(kwargs_replica)
burst_id, _ = self.burst_service.launch_burst(burst_config, 0,
first_step_algo.id,
self.test_user.id)
burst_config = dao.get_burst_by_id(burst_id)
self.assertTrue(
burst_config.status in (BurstConfiguration.BURST_FINISHED,
BurstConfiguration.BURST_RUNNING),
"Burst not launched successfully!")
# Wait maximum x seconds for burst to finish
self._wait_for_burst(burst_config)
def _add_new_valid_portlets(self, portlets_list):
# Now add portlets that were not in DB at previous run but are valid now
for portlet in portlets_list:
db_entity = dao.get_portlet_by_identifier(portlet.algorithm_identifier)
if db_entity is None:
self.logger.debug("Will now store portlet %s" % (str(portlet),))
dao.store_entity(portlet)
def test_launch_burst_invalid_portlet_analyzer_data(self):
"""
Test that burst is marked as error if invalid data is passed to the first step.
"""
algo_id = self.flow_service.get_algorithm_by_module_and_class('tvb.tests.framework.adapters.testadapter1',
'TestAdapter1').id
#Adapter tries to do an int(test1_val1) and int(test1_val2) so this should be valid
burst_config = self.burst_service.new_burst_configuration(self.test_project.id)
kwargs_replica = {'test1_val1': '1', 'test1_val2': '0'}
burst_config.update_simulator_configuration(kwargs_replica)
test_portlet = dao.get_portlet_by_identifier(self.PORTLET_ID)
portlet_configuration = self.burst_service.new_portlet_configuration(test_portlet.id)
#Portlet analyzer tries to do int(input) which should fail
declared_overwrites = {ADAPTER_PREFIX_ROOT + '0test_non_dt_input': 'asa'}
self.burst_service.update_portlet_configuration(portlet_configuration, declared_overwrites)
burst_config.tabs[0].portlets[0] = portlet_configuration
burst_id, _ = self.burst_service.launch_burst(burst_config, 0, algo_id, self.test_user.id)
burst_config = dao.get_burst_by_id(burst_id)
#Wait maximum x seconds for burst to finish
burst_config = self._wait_for_burst(burst_config, error_expected=True)
burst_wf = dao.get_workflows_for_burst(burst_config.id)[0]
wf_steps = dao.get_workflow_steps(burst_wf.id)
self.assertTrue(len(wf_steps) == 2,
"Should have exactly 2 wf steps. One for 'simulation' one for portlet analyze operation.")
simulator_op = dao.get_operation_by_id(wf_steps[0].fk_operation)
self.assertEqual(model.STATUS_FINISHED, simulator_op.status,
"First operation should be simulator which should have 'finished' status.")
portlet_analyze_op = dao.get_operation_by_id(wf_steps[1].fk_operation)
self.assertEqual(portlet_analyze_op.status, model.STATUS_ERROR,
"Second operation should be portlet analyze step which should have 'error' status.")
def test_update_portlet_config(self):
"""
Test if a portlet configuration parameters are updated accordingly with a set
of overwrites that would normally come from UI. Make sure to restart only if
analyzer parameters change.
"""
def __update_params(declared_overwrites, expected_result):
"""
Do the update and check that we get indeed the expected_result.
:param declared_overwrites: a input dictionary in the form {'$$name$$' : '$$value$$'}. Make
sure $$name$$ has the prefix that is added in case of portlet parameters,
namely ADAPTER_PREFIX_ROOT + step_index + actual_name
:param expected_result: boolean which should represent if we need or not to restart. (Was a
visualizer parameter change or an analyzer one)
"""
result = self.burst_service.update_portlet_configuration(portlet_configuration, declared_overwrites)
self.assertEqual(expected_result, result,
"After update expected %s as 'need_restart' but got %s." % (expected_result, result))
test_portlet = dao.get_portlet_by_identifier(self.PORTLET_ID)
portlet_configuration = self.burst_service.new_portlet_configuration(test_portlet.id)
previous_entry = portlet_configuration.analyzers[0].static_param['test_non_dt_input']
declared_overwrites = {ADAPTER_PREFIX_ROOT + '0test_non_dt_input': previous_entry}
__update_params(declared_overwrites, False)
declared_overwrites = {ADAPTER_PREFIX_ROOT + '1test2': 'new_value'}
__update_params(declared_overwrites, False)
declared_overwrites = {ADAPTER_PREFIX_ROOT + '0test_non_dt_input': '1'}
__update_params(declared_overwrites, True)
def test_launch_group_burst_no_metric(self):
"""
Test the launch burst method from burst service. Try to launch a burst with test adapter which has
no metrics associated. This should fail.
"""
burst_config = self.burst_service.new_burst_configuration(self.test_project.id)
algo_id = self.flow_service.get_algorithm_by_module_and_class('tvb.tests.framework.adapters.testadapter1',
'TestAdapter1').id
kwargs_replica = {'test1_val1': '[0, 1, 2]', 'test1_val2': '0', model.RANGE_PARAMETER_1: 'test1_val1'}
test_portlet = dao.get_portlet_by_identifier(self.PORTLET_ID)
tab_config = {test_portlet.id: [(0, 0), (0, 1), (1, 0)]}
self._add_portlets_to_burst(burst_config, tab_config)
burst_config.update_simulator_configuration(kwargs_replica)
burst_id, _ = self.burst_service.launch_burst(burst_config, 0, algo_id, self.test_user.id)
burst_config = dao.get_burst_by_id(burst_id)
# Wait maximum x seconds for burst to finish
self._wait_for_burst(burst_config, error_expected=True)
launched_workflows = dao.get_workflows_for_burst(burst_id, is_count=True)
self.assertEqual(3, launched_workflows, "3 workflows should have been launched due to group parameter.")
op_groups = self.count_all_entities(model.OperationGroup)
dt_groups = self.count_all_entities(model.DataTypeGroup)
self.assertEqual(5, op_groups, "An operation group should have been created for each step.")
self.assertEqual(5, dt_groups, "An dataType group should have been created for each step.")
def get_portlet(self):
"""
Return the portlet that was saved on this entity. Should only be called if
beforehand 'set_portlet' was called on this entity.
"""
if self.PORTLET_IDENT not in self._data_dict:
return None
return dao.get_portlet_by_identifier(self._data_dict[self.PORTLET_IDENT])
def get_portlet(self):
"""
Return the portlet that was saved on this entity. Should only be called if
beforehand 'set_portlet' was called on this entity.
"""
if self.PORTLET_IDENT not in self._data_dict:
return None
return dao.get_portlet_by_identifier(self._data_dict[self.PORTLET_IDENT])
def test_build_portlet_interface_invalid(self):
"""
Test that a proper exception is raised in case an invalid portlet configuration is provided.
"""
test_portlet = dao.get_portlet_by_identifier(self.PORTLET_ID)
portlet_configuration = self.burst_service.new_portlet_configuration(test_portlet.id)
portlet_configuration.portlet_id = "this-is-invalid"
self.assertRaises(InvalidPortletConfiguration, self.burst_service.build_portlet_interface,
portlet_configuration, self.test_project.id)
def test_portlet_tab_display(self):
"""
Update the default portlet configuration, by storing a TimeSeries
portlet for all postions. Then check that we get the same configuration.
"""
self.burst_c.index()
portlet_id = dao.get_portlet_by_identifier("TimeSeries").id
one_tab = [[portlet_id, "TimeSeries"] for _ in range(NUMBER_OF_PORTLETS_PER_TAB)]
full_tabs = [one_tab for _ in range(BurstConfiguration.nr_of_tabs)]
data = {'tab_portlets_list': json.dumps(full_tabs)}
result = self.burst_c.portlet_tab_display(**data)
selected_portlets = result['portlet_tab_list']
for entry in selected_portlets:
self.assertEqual(entry.id, portlet_id)
def test_portlet_configurable_interface(self):
"""
A simple test for the get configurable interface method.
"""
test_portlet = dao.get_portlet_by_identifier("TA1TA2")
result = PortletConfigurer(test_portlet).get_configurable_interface()
self.assertEqual(len(result), 2, "Length of the resulting interface not as expected")
for one_entry in result:
for entry in one_entry.interface:
if entry['name'] == 'test1':
self.assertTrue(entry['default'] == 'step_0[0]', "Overwritten default not in effect.")
if entry['name'] == 'test2':
self.assertTrue(entry['default'] == '0', "Value that was not overwritten changed.")
def test_get_portlet_session_configuration(self):
"""
Test that the default portlet session sonciguration is generated
as expected, with a default TimeSeries portlet and rest empty.
"""
self.burst_c.index()
result = json.loads(self.burst_c.get_portlet_session_configuration())
portlet_id = dao.get_portlet_by_identifier("TimeSeries").id
for tab_idx, tab in enumerate(result):
for index_in_tab, value in enumerate(tab):
if tab_idx == 0 and index_in_tab == 0:
self.assertEqual(value, [portlet_id, "TimeSeries"])
else:
self.assertEqual(value, [-1, "None"])
def test_portlet_configurable_interface(self):
"""
A simple test for the get configurable interface method.
"""
test_portlet = dao.get_portlet_by_identifier("TA1TA2")
result = PortletConfigurer(test_portlet).get_configurable_interface()
self.assertEqual(len(result), 2, "Length of the resulting interface not as expected")
for one_entry in result:
for entry in one_entry.interface:
if entry['name'] == 'test1':
self.assertTrue(entry['default'] == 'step_0[0]', "Overwritten default not in effect.")
if entry['name'] == 'test2':
self.assertTrue(entry['default'] == '0', "Value that was not overwritten changed.")
def test_get_portlet_session_configuration(self):
"""
Test that the default portlet session sonciguration is generated
as expected, with a default TimeSeries portlet and rest empty.
"""
self.burst_c.index()
result = json.loads(self.burst_c.get_portlet_session_configuration())
portlet_id = dao.get_portlet_by_identifier("TimeSeries").id
for tab_idx, tab in enumerate(result):
for index_in_tab, value in enumerate(tab):
if tab_idx == 0 and index_in_tab == 0:
self.assertEqual(value, [portlet_id, "TimeSeries"])
else:
self.assertEqual(value, [-1, "None"])
def test_portlet_tab_display(self):
"""
Update the default portlet configuration, by storing a TimeSeries
portlet for all postions. Then check that we get the same configuration.
"""
self.burst_c.index()
portlet_id = dao.get_portlet_by_identifier("TimeSeries").id
one_tab = [[portlet_id, "TimeSeries"] for _ in range(NUMBER_OF_PORTLETS_PER_TAB)]
full_tabs = [one_tab for _ in range(BurstConfiguration.nr_of_tabs)]
data = {'tab_portlets_list': json.dumps(full_tabs)}
result = self.burst_c.portlet_tab_display(**data)
selected_portlets = result['portlet_tab_list']
for entry in selected_portlets:
self.assertEqual(entry.id, portlet_id)
def _prepare_and_launch_sync_burst(self):
"""
Private method to launch a dummy burst. Return the burst loaded after the launch finished
as well as the workflow steps that initially formed the burst.
NOTE: the burst launched by this method is a `dummy` one, meaning we do not use an actual
simulation, but instead test adapters.
"""
burst_config = TestFactory.store_burst(self.test_project.id)
workflow_step_list = []
test_portlet = dao.get_portlet_by_identifier(self.PORTLET_ID)
stored_dt = datatypes_factory.DatatypesFactory()._store_datatype(
Datatype1())
first_step_algorithm = self.flow_service.get_algorithm_by_module_and_class(
"tvb.tests.framework.adapters.testadapter1",
"TestAdapterDatatypeInput")
metadata = {DataTypeMetaData.KEY_BURST: burst_config.id}
kwargs = {"test_dt_input": stored_dt.gid, 'test_non_dt_input': '0'}
operations, group = self.operation_service.prepare_operations(
self.test_user.id, self.test_project.id, first_step_algorithm,
first_step_algorithm.algorithm_category, metadata, **kwargs)
view_step = TestFactory.create_workflow_step(
"tvb.tests.framework.adapters.testadapter2",
"TestAdapter2", {"test2": 2}, {},
0,
0,
0,
0,
is_view_step=True)
view_step.fk_portlet = test_portlet.id
workflow_step_list.append(view_step)
workflows = self.workflow_service.create_and_store_workflow(
self.test_project.id, burst_config.id, 0, first_step_algorithm.id,
operations)
self.operation_service.prepare_operations_for_workflowsteps(
workflow_step_list, workflows, self.test_user.id, burst_config.id,
self.test_project.id, group, operations)
### Now fire the workflow and also update and store the burst configuration ##
self.operation_service.launch_operation(operations[0].id, False)
loaded_burst, _ = self.burst_service.load_burst(burst_config.id)
import_operation = dao.get_operation_by_id(stored_dt.fk_from_operation)
dao.remove_entity(import_operation.__class__, import_operation.id)
dao.remove_datatype(stored_dt.gid)
return loaded_burst, workflow_step_list
def test_update_portlet_config_invalid_data(self):
"""
Trying an update on a portlet configuration with invalid data
should not change the configuration instance in any way.
"""
test_portlet = dao.get_portlet_by_identifier(self.PORTLET_ID)
portlet_configuration = self.burst_service.new_portlet_configuration(test_portlet.id)
invalid_overwrites = {'this_is_not_a_valid_key': 'for_test_portlet_update'}
before_update = copy.deepcopy(portlet_configuration)
self.burst_service.update_portlet_configuration(portlet_configuration, invalid_overwrites)
self.assertEqual(set(dir(before_update)), set(dir(portlet_configuration)))
#An update with invalid input data should have no effect on the configuration, but attributes changed
for key in portlet_configuration.__dict__.keys():
if hasattr(getattr(portlet_configuration, key), '__call__'):
self.assertEqual(getattr(before_update, key), getattr(portlet_configuration, key),
"The value of attribute %s changed by a update with invalid data "
"when it shouldn't have." % key)
def test_new_portlet_configuration(self):
"""
Test that the correct portlet configuration is generated for the test portlet.
"""
# Passing an invalid portlet ID should fail and raise an InvalidPortletConfiguration exception.
self.assertRaises(InvalidPortletConfiguration, self.burst_service.new_portlet_configuration, -1)
# Now the happy flow
test_portlet = dao.get_portlet_by_identifier(self.PORTLET_ID)
portlet_configuration = self.burst_service.new_portlet_configuration(test_portlet.id)
analyzers = portlet_configuration.analyzers
self.assertEqual(len(analyzers), 1, "Portlet configuration not build properly."
" Portlet's analyzers list has unexpected number of elements.")
self.assertEqual(analyzers[0].dynamic_param, {u'test_dt_input': {wf_cfg.DATATYPE_INDEX_KEY: 0,
wf_cfg.STEP_INDEX_KEY: 0}},
"Dynamic parameters not loaded properly")
visualizer = portlet_configuration.visualizer
self.assertEqual(visualizer.dynamic_param, {}, "Dynamic parameters not loaded properly")
self.assertEqual(visualizer.static_param, {u'test2': u'0'}, 'Static parameters not loaded properly')
def test_build_portlet_interface(self):
"""
Test that the portlet interface is build properly, splitted by steps and prefixed.
"""
test_portlet = dao.get_portlet_by_identifier(self.PORTLET_ID)
portlet_configuration = self.burst_service.new_portlet_configuration(
test_portlet.id)
actual_interface = self.burst_service.build_portlet_interface(
portlet_configuration, self.test_project.id)
#The expected portlet steps and interface in correspondace to the xml declaration
#from tvb.tests.framework/core/portlets/test_portlet.xml
expected_steps = [{
'ui_name': 'TestAdapterDatatypeInput'
}, {
'ui_name': 'TestAdapter2'
}]
expected_interface = [{
ABCAdapter.KEY_DEFAULT:
'step_0[0]',
ABCAdapter.KEY_DISABLED:
True,
KEY_DYNAMIC:
True,
ABCAdapter.KEY_NAME:
ADAPTER_PREFIX_ROOT + '0test_dt_input'
}, {
ABCAdapter.KEY_DEFAULT:
'0',
ABCAdapter.KEY_DISABLED:
False,
KEY_DYNAMIC:
False,
ABCAdapter.KEY_NAME:
ADAPTER_PREFIX_ROOT + '1test2'
}]
for idx, entry in enumerate(expected_steps):
step = actual_interface[idx]
for key in entry:
self.assertEqual(entry.get(key), getattr(step, key))
for key in expected_interface[idx]:
self.assertEqual(expected_interface[idx].get(key, False),
step.interface[0].get(key, False))
def test_index(self):
"""
Test that index returns a dict with all required keys. Also check
that the default portlets are populated, with only the first being
the TimeSeries portlet and the rest are empty.
"""
result_dict = self.burst_c.index()
self.assertTrue('burst_list' in result_dict and result_dict['burst_list'] == [])
self.assertTrue('available_metrics' in result_dict and isinstance(result_dict['available_metrics'], list))
self.assertTrue('portletList' in result_dict and isinstance(result_dict['portletList'], list))
self.assertEqual(result_dict[common.KEY_SECTION], "burst")
self.assertTrue('burstConfig' in result_dict and isinstance(result_dict['burstConfig'], BurstConfiguration))
portlets = json.loads(result_dict['selectedPortlets'])
portlet_id = dao.get_portlet_by_identifier("TimeSeries").id
for tab_idx, tab in enumerate(portlets):
for index_in_tab, value in enumerate(tab):
if tab_idx == 0 and index_in_tab == 0:
self.assertEqual(value, [portlet_id, "TimeSeries"])
else:
self.assertEqual(value, [-1, "None"])
self.assertTrue(result_dict['draw_hidden_ranges'])
def test_build_portlet_interface(self):
"""
Test that the portlet interface is build properly, splitted by steps and prefixed.
"""
test_portlet = dao.get_portlet_by_identifier(self.PORTLET_ID)
portlet_configuration = self.burst_service.new_portlet_configuration(test_portlet.id)
actual_interface = self.burst_service.build_portlet_interface(portlet_configuration, self.test_project.id)
#The expected portlet steps and interface in correspondace to the xml declaration
#from tvb.tests.framework/core/portlets/test_portlet.xml
expected_steps = [{'ui_name': 'TestAdapterDatatypeInput'},
{'ui_name': 'TestAdapter2'}]
expected_interface = [{ABCAdapter.KEY_DEFAULT: 'step_0[0]', ABCAdapter.KEY_DISABLED: True,
KEY_DYNAMIC: True, ABCAdapter.KEY_NAME: ADAPTER_PREFIX_ROOT + '0test_dt_input'},
{ABCAdapter.KEY_DEFAULT: '0', ABCAdapter.KEY_DISABLED: False,
KEY_DYNAMIC: False, ABCAdapter.KEY_NAME: ADAPTER_PREFIX_ROOT + '1test2'}]
for idx, entry in enumerate(expected_steps):
step = actual_interface[idx]
for key in entry:
self.assertEqual(entry.get(key), getattr(step, key))
for key in expected_interface[idx]:
self.assertEqual(expected_interface[idx].get(key, False), step.interface[0].get(key, False))
def _prepare_and_launch_sync_burst(self):
"""
Private method to launch a dummy burst. Return the burst loaded after the launch finished
as well as the workflow steps that initially formed the burst.
NOTE: the burst launched by this method is a `dummy` one, meaning we do not use an actual
simulation, but instead test adapters.
"""
burst_config = TestFactory.store_burst(self.test_project.id)
workflow_step_list = []
test_portlet = dao.get_portlet_by_identifier(self.PORTLET_ID)
stored_dt = datatypes_factory.DatatypesFactory()._store_datatype(Datatype1())
first_step_algorithm = self.flow_service.get_algorithm_by_module_and_class(
"tvb.tests.framework.adapters.testadapter1", "TestAdapterDatatypeInput")
metadata = {DataTypeMetaData.KEY_BURST: burst_config.id}
kwargs = {"test_dt_input": stored_dt.gid, 'test_non_dt_input': '0'}
operations, group = self.operation_service.prepare_operations(self.test_user.id, self.test_project.id,
first_step_algorithm,
first_step_algorithm.algorithm_category,
metadata, **kwargs)
view_step = TestFactory.create_workflow_step("tvb.tests.framework.adapters.testadapter2", "TestAdapter2",
{"test2": 2}, {}, 0, 0, 0, 0, is_view_step=True)
view_step.fk_portlet = test_portlet.id
workflow_step_list.append(view_step)
workflows = self.workflow_service.create_and_store_workflow(self.test_project.id, burst_config.id, 0,
first_step_algorithm.id, operations)
self.operation_service.prepare_operations_for_workflowsteps(workflow_step_list, workflows, self.test_user.id,
burst_config.id, self.test_project.id, group,
operations)
### Now fire the workflow and also update and store the burst configuration ##
self.operation_service.launch_operation(operations[0].id, False)
loaded_burst, _ = self.burst_service.load_burst(burst_config.id)
import_operation = dao.get_operation_by_id(stored_dt.fk_from_operation)
dao.remove_entity(import_operation.__class__, import_operation.id)
dao.remove_datatype(stored_dt.gid)
return loaded_burst, workflow_step_list
def __get_portlets(self, path_portlets):
"""
Given a path in the form of a python package e.g.: "tvb.portlets', import
the package, get it's folder and look for all the XML files defined
there, then read all the portlets defined there and store them in DB.
"""
portlet_package = __import__(path_portlets, globals(), locals(),
["__init__"])
portlet_folder = os.path.dirname(portlet_package.__file__)
portlets_list = []
for file_n in os.listdir(portlet_folder):
try:
if file_n.endswith('.xml'):
complete_file_path = os.path.join(portlet_folder, file_n)
portlet_reader = XMLPortletReader(complete_file_path)
portlet_list = portlet_reader.get_algorithms_dictionary()
self.logger.debug(
"Starting to verify currently declared portlets in %s."
% (file_n, ))
for algo_identifier in portlet_list:
adapters_chain = portlet_reader.get_adapters_chain(
algo_identifier)
is_valid = True
for adapter in adapters_chain:
class_name = adapter[ABCAdapter.KEY_TYPE].split(
'.')[-1]
module_name = adapter[ABCAdapter.KEY_TYPE].replace(
'.' + class_name, '')
try:
#Check that module is properly declared
module = __import__(module_name,
globals(),
fromlist=[class_name])
if type(module) != ModuleType:
is_valid = False
self.logger.error(
"Wrong module %s in portlet %s" %
(module_name, algo_identifier))
continue
#Check that class is properly declared
if not hasattr(module, class_name):
is_valid = False
self.logger.error(
"Wrong class %s in portlet %s." %
(class_name, algo_identifier))
continue
#Check inputs that refers to this adapter
portlet_inputs = portlet_list[algo_identifier][
ELEM_INPUTS]
adapter_instance = PortletConfigurer.build_adapter_from_declaration(
adapter)
if adapter_instance is None:
is_valid = False
self.logger.warning(
"No group having class=%s stored for "
"portlet %s." %
(class_name, algo_identifier))
continue
adapter_input_names = [
entry[ABCAdapter.KEY_NAME] for entry in
adapter_instance.flaten_input_interface()
]
for input_entry in portlet_inputs.values():
if input_entry[ATT_OVERWRITE] == adapter[
ABCAdapter.KEY_NAME]:
if input_entry[
ABCAdapter.
KEY_NAME] not in adapter_input_names:
self.logger.error(
"Invalid input %s for adapter %s"
% (input_entry[
ABCAdapter.KEY_NAME],
adapter_instance))
is_valid = False
except ImportError:
self.logger.error(
"Invalid adapter declaration %s in portlet %s"
% (adapter[ABCAdapter.KEY_TYPE],
algo_identifier))
is_valid = False
if is_valid:
portlets_list.append(
model.Portlet(
algo_identifier, complete_file_path,
portlet_list[algo_identifier]['name']))
except XmlParserException as excep:
self.logger.exception(excep)
self.logger.error("Invalid Portlet description File " +
file_n + " will continue without it!!")
self.logger.debug("Refreshing portlets from xml declarations.")
stored_portlets = dao.get_available_portlets()
#First update old portlets from DB
for stored_portlet in stored_portlets:
for verified_portlet in portlets_list:
if stored_portlet.algorithm_identifier == verified_portlet.algorithm_identifier:
stored_portlet.xml_path = verified_portlet.xml_path
stored_portlet.last_introspection_check = datetime.datetime.now(
)
stored_portlet.name = verified_portlet.name
dao.store_entity(stored_portlet)
break
#Now add portlets that were not in DB at previous run but are valid now
for portlet in portlets_list:
db_entity = dao.get_portlet_by_identifier(
portlet.algorithm_identifier)
if db_entity is None:
self.logger.debug("Will now store portlet %s" %
(str(portlet), ))
dao.store_entity(portlet)
self.logger.debug("Refreshing portlets from xml declarations.")
stored_portlets = dao.get_available_portlets()
#First update old portlets from DB
for stored_portlet in stored_portlets:
for verified_portlet in portlets_list:
if stored_portlet.algorithm_identifier == verified_portlet.algorithm_identifier:
stored_portlet.xml_path = verified_portlet.xml_path
stored_portlet.last_introspection_check = datetime.datetime.now(
)
stored_portlet.name = verified_portlet.name
dao.store_entity(stored_portlet)
break
#Now add portlets that were not in DB at previous run but are valid now
for portlet in portlets_list:
db_entity = dao.get_portlet_by_identifier(
portlet.algorithm_identifier)
if db_entity is None:
self.logger.debug("Will now store portlet %s" %
(str(portlet), ))
dao.store_entity(portlet)
def __get_datatypes(self, path_types):
"""
Imports each DataType to update the DB model, by creating a new table for each DataType.
"""
for my_type in Introspector.__get_variable(path_types):
try:
module_ref = __import__(path_types, globals(), locals(),
[my_type])
module_ref = eval("module_ref." + my_type)
tree = [
def get_portlet_by_identifier(portlet_identifier):
"""
:returns: the portlet entity with the algorithm identifier =@portlet_identifier
"""
return dao.get_portlet_by_identifier(portlet_identifier)
def get_portlet_by_identifier(portlet_identifier):
"""
:returns: the portlet entity with the algorithm identifier =@portlet_identifier
"""
return dao.get_portlet_by_identifier(portlet_identifier)
def __get_portlets(self, path_portlets):
"""
Given a path in the form of a python package e.g.: "tvb.portlets', import
the package, get it's folder and look for all the XML files defined
there, then read all the portlets defined there and store them in DB.
"""
portlet_package = __import__(path_portlets, globals(), locals(), ["__init__"])
portlet_folder = os.path.dirname(portlet_package.__file__)
portlets_list = []
for file_n in os.listdir(portlet_folder):
try:
if file_n.endswith('.xml'):
complete_file_path = os.path.join(portlet_folder, file_n)
portlet_reader = XMLPortletReader(complete_file_path)
portlet_list = portlet_reader.get_algorithms_dictionary()
self.logger.debug("Starting to verify currently declared portlets in %s." % (file_n,))
for algo_identifier in portlet_list:
adapters_chain = portlet_reader.get_adapters_chain(algo_identifier)
is_valid = True
for adapter in adapters_chain:
class_name = adapter[ABCAdapter.KEY_TYPE].split('.')[-1]
module_name = adapter[ABCAdapter.KEY_TYPE].replace('.' + class_name, '')
try:
#Check that module is properly declared
module = __import__(module_name, globals(), fromlist=[class_name])
if type(module) != ModuleType:
is_valid = False
self.logger.error("Wrong module %s in portlet %s" % (module_name, algo_identifier))
continue
#Check that class is properly declared
if not hasattr(module, class_name):
is_valid = False
self.logger.error("Wrong class %s in portlet %s." % (class_name, algo_identifier))
continue
#Check inputs that refers to this adapter
portlet_inputs = portlet_list[algo_identifier][ELEM_INPUTS]
adapter_instance = PortletConfigurer.build_adapter_from_declaration(adapter)
if adapter_instance is None:
is_valid = False
self.logger.warning("No group having class=%s stored for "
"portlet %s." % (class_name, algo_identifier))
continue
adapter_input_names = [entry[ABCAdapter.KEY_NAME] for entry
in adapter_instance.flaten_input_interface()]
for input_entry in portlet_inputs.values():
if input_entry[ATT_OVERWRITE] == adapter[ABCAdapter.KEY_NAME]:
if input_entry[ABCAdapter.KEY_NAME] not in adapter_input_names:
self.logger.error("Invalid input %s for adapter %s" % (
input_entry[ABCAdapter.KEY_NAME], adapter_instance))
is_valid = False
except ImportError:
self.logger.error("Invalid adapter declaration %s in portlet %s" % (
adapter[ABCAdapter.KEY_TYPE], algo_identifier))
is_valid = False
if is_valid:
portlets_list.append(model.Portlet(algo_identifier, complete_file_path,
portlet_list[algo_identifier]['name']))
except XmlParserException as excep:
self.logger.exception(excep)
self.logger.error("Invalid Portlet description File " + file_n + " will continue without it!!")
self.logger.debug("Refreshing portlets from xml declarations.")
stored_portlets = dao.get_available_portlets()
#First update old portlets from DB
for stored_portlet in stored_portlets:
for verified_portlet in portlets_list:
if stored_portlet.algorithm_identifier == verified_portlet.algorithm_identifier:
stored_portlet.xml_path = verified_portlet.xml_path
stored_portlet.last_introspection_check = datetime.datetime.now()
stored_portlet.name = verified_portlet.name
dao.store_entity(stored_portlet)
break
#Now add portlets that were not in DB at previous run but are valid now
for portlet in portlets_list:
db_entity = dao.get_portlet_by_identifier(portlet.algorithm_identifier)
if db_entity is None:
self.logger.debug("Will now store portlet %s" % (str(portlet),))
dao.store_entity(portlet)
self.logger.debug("Refreshing portlets from xml declarations.")
stored_portlets = dao.get_available_portlets()
#First update old portlets from DB
for stored_portlet in stored_portlets:
for verified_portlet in portlets_list:
if stored_portlet.algorithm_identifier == verified_portlet.algorithm_identifier:
stored_portlet.xml_path = verified_portlet.xml_path
stored_portlet.last_introspection_check = datetime.datetime.now()
stored_portlet.name = verified_portlet.name
dao.store_entity(stored_portlet)
break
#Now add portlets that were not in DB at previous run but are valid now
for portlet in portlets_list:
db_entity = dao.get_portlet_by_identifier(portlet.algorithm_identifier)
if db_entity is None:
self.logger.debug("Will now store portlet %s" % (str(portlet),))
dao.store_entity(portlet)
@staticmethod
def _is_concrete_subclass(clz, super_cls):
return inspect.isclass(clz) and not inspect.isabstract(clz) and issubclass(clz, super_cls)
def __get_datatypes(self, path_types):
"""
Imports each DataType to update the DB model, by creating a new table for each DataType.
"""
for my_type in Introspector.__read_module_variable(path_types):
try:
