def __get_portlets(self, path_portlets):
"""
Given a path in the form of a python package e.g.: "tvb.portlets', import
the package, get it's folder and look for all the XML files defined
there, then read all the portlets defined there and store them in DB.
"""
portlet_package = __import__(path_portlets, globals(), locals(), ["__init__"])
portlet_folder = os.path.dirname(portlet_package.__file__)
portlets_list = []
for file_n in os.listdir(portlet_folder):
try:
if file_n.endswith('.xml'):
complete_file_path = os.path.join(portlet_folder, file_n)
portlet_reader = XMLPortletReader(complete_file_path)
portlet_list = portlet_reader.get_algorithms_dictionary()
self.logger.debug("Starting to verify currently declared portlets in %s." % (file_n,))
for algo_identifier in portlet_list:
adapters_chain = portlet_reader.get_adapters_chain(algo_identifier)
is_valid = True
for adapter in adapters_chain:
class_name = adapter[ABCAdapter.KEY_TYPE].split('.')[-1]
module_name = adapter[ABCAdapter.KEY_TYPE].replace('.' + class_name, '')
try:
#Check that module is properly declared
module = __import__(module_name, globals(), fromlist=[class_name])
if type(module) != ModuleType:
is_valid = False
self.logger.error("Wrong module %s in portlet %s" % (module_name, algo_identifier))
continue
#Check that class is properly declared
if not hasattr(module, class_name):
is_valid = False
self.logger.error("Wrong class %s in portlet %s." % (class_name, algo_identifier))
continue
#Check inputs that refers to this adapter
portlet_inputs = portlet_list[algo_identifier][ELEM_INPUTS]
adapter_instance = PortletConfigurer.build_adapter_from_declaration(adapter)
if adapter_instance is None:
is_valid = False
self.logger.warning("No group having class=%s stored for "
"portlet %s." % (class_name, algo_identifier))
continue
adapter_input_names = [entry[ABCAdapter.KEY_NAME] for entry
in adapter_instance.flaten_input_interface()]
for input_entry in portlet_inputs.values():
if input_entry[ATT_OVERWRITE] == adapter[ABCAdapter.KEY_NAME]:
if input_entry[ABCAdapter.KEY_NAME] not in adapter_input_names:
self.logger.error("Invalid input %s for adapter %s" % (
input_entry[ABCAdapter.KEY_NAME], adapter_instance))
is_valid = False
except ImportError, _:
self.logger.error("Invalid adapter declaration %s in portlet %s" % (
adapter[ABCAdapter.KEY_TYPE], algo_identifier))
is_valid = False
if is_valid:
portlets_list.append(model.Portlet(algo_identifier, complete_file_path,
portlet_list[algo_identifier]['name']))
except XmlParserException, excep:
self.logger.exception(excep)
self.logger.error("Invalid Portlet description File " + file_n + " will continue without it!!")
def __get_portlets(self, path_portlets):
"""
Given a path in the form of a python package e.g.: "tvb.portlets', import
the package, get it's folder and look for all the XML files defined
there, then read all the portlets defined there and store them in DB.
"""
portlet_package = __import__(path_portlets, globals(), locals(), ["__init__"])
portlet_folder = os.path.dirname(portlet_package.__file__)
portlets_list = []
for file_n in os.listdir(portlet_folder):
try:
if file_n.endswith('.xml'):
complete_file_path = os.path.join(portlet_folder, file_n)
portlet_reader = XMLPortletReader(complete_file_path)
portlet_list = portlet_reader.get_algorithms_dictionary()
self.logger.debug("Starting to verify currently declared portlets in %s." % (file_n,))
for algo_identifier in portlet_list:
adapters_chain = portlet_reader.get_adapters_chain(algo_identifier)
is_valid = True
for adapter in adapters_chain:
class_name = adapter[ABCAdapter.KEY_TYPE].split('.')[-1]
module_name = adapter[ABCAdapter.KEY_TYPE].replace('.' + class_name, '')
try:
#Check that module is properly declared
module = __import__(module_name, globals(), fromlist=[class_name])
if type(module) != ModuleType:
is_valid = False
self.logger.error("Wrong module %s in portlet %s" % (module_name, algo_identifier))
continue
#Check that class is properly declared
if not hasattr(module, class_name):
is_valid = False
self.logger.error("Wrong class %s in portlet %s." % (class_name, algo_identifier))
continue
#Check inputs that refers to this adapter
portlet_inputs = portlet_list[algo_identifier][ELEM_INPUTS]
adapter_instance = PortletConfigurer.build_adapter_from_declaration(adapter)
if adapter_instance is None:
is_valid = False
self.logger.warning("No group having class=%s stored for "
"portlet %s." % (class_name, algo_identifier))
continue
adapter_input_names = [entry[ABCAdapter.KEY_NAME] for entry
in adapter_instance.flaten_input_interface()]
for input_entry in portlet_inputs.values():
if input_entry[ATT_OVERWRITE] == adapter[ABCAdapter.KEY_NAME]:
if input_entry[ABCAdapter.KEY_NAME] not in adapter_input_names:
self.logger.error("Invalid input %s for adapter %s" % (
input_entry[ABCAdapter.KEY_NAME], adapter_instance))
is_valid = False
except ImportError, _:
self.logger.error("Invalid adapter declaration %s in portlet %s" % (
adapter[ABCAdapter.KEY_TYPE], algo_identifier))
is_valid = False
if is_valid:
portlets_list.append(model.Portlet(algo_identifier, complete_file_path,
portlet_list[algo_identifier]['name']))
except XmlParserException, excep:
self.logger.exception(excep)
self.logger.error("Invalid Portlet description File " + file_n + " will continue without it!!")
def _prepare_valid_portlets_list(self, portlet_folder, portlets_list):
for file_n in os.listdir(portlet_folder):
try:
if file_n.endswith('.xml'):
complete_file_path = os.path.join(portlet_folder, file_n)
portlet_reader = XMLPortletReader(complete_file_path)
portlet_list = portlet_reader.get_algorithms_dictionary()
self.logger.debug("Starting to verify currently declared portlets in %s." % (file_n,))
for algo_identifier in portlet_list:
adapters_chain = portlet_reader.get_adapters_chain(algo_identifier)
is_valid = True
for adapter in adapters_chain:
class_name = adapter[ABCAdapter.KEY_TYPE].split('.')[-1]
module_name = adapter[ABCAdapter.KEY_TYPE].replace('.' + class_name, '')
try:
# Check that module is properly declared
module = importlib.import_module(module_name)
if type(module) != ModuleType:
is_valid = False
self.logger.error("Wrong module %s in portlet %s" % (module_name, algo_identifier))
continue
# Check that class is properly declared
if not hasattr(module, class_name):
is_valid = False
self.logger.error("Wrong class %s in portlet %s." % (class_name, algo_identifier))
continue
# Check inputs that refers to this adapter
portlet_inputs = portlet_list[algo_identifier][ELEM_INPUTS]
adapter_instance = self._build_adapter_from_declaration(adapter)
if adapter_instance is None:
is_valid = False
self.logger.warning("No group having class=%s stored for portlet %s."
% (class_name, algo_identifier))
continue
adapter_form = adapter_instance.get_form()
adapter_instance.submit_form(adapter_form())
# TODO: implement this for neoforms
adapter_form_field_names = {} # adapter_instance.flaten_input_interface()
for input_entry in portlet_inputs.values():
if input_entry[ATT_OVERWRITE] == adapter[ABCAdapter.KEY_NAME]:
if input_entry[ABCAdapter.KEY_NAME] not in adapter_form_field_names:
self.logger.error("Invalid input %s for adapter %s"
% (input_entry[ABCAdapter.KEY_NAME], adapter_instance))
is_valid = False
except ImportError:
is_valid = False
self.logger.error("Invalid adapter declaration %s in portlet %s"
% (adapter[ABCAdapter.KEY_TYPE], algo_identifier))
if is_valid:
portlets_list.append(
Portlet(algo_identifier, complete_file_path, portlet_list[algo_identifier]['name']))
except XmlParserException as excep:
self.logger.exception(excep)
self.logger.error("Invalid Portlet description File " + file_n + " will continue without it!!")
class PortletConfigurer(object):
"""
Helper class that handles all the functionality required from a portlet.
Given a portlet entity, this will allow the following:
- return a configurable interface in the form of a dictionary
equivalent to the input tree of an adapter
- create a new PortletConfiguration entity, or update a already
created one with a new set of parameters
"""
def __init__(self, portlet_entity):
self.log = get_logger(self.__class__.__module__)
self.reader = XMLPortletReader(portlet_entity.xml_path)
self.portlet_entity = portlet_entity
@property
def portlet_id(self):
"""Portlet DB identifier"""
return self.portlet_entity.id
@property
def algo_identifier(self):
"""Unique identifier for current portlet."""
return self.portlet_entity.algorithm_identifier
@property
def ui_name(self):
""" Portlet name to be displayed in UI"""
return self.portlet_entity.name
def get_configurable_interface(self):
"""
Given an algorithm identifier, go trough the adapter chain, and merge
their input tree with the declared overwrites
"""
chain_adapters = self.reader.get_adapters_chain(self.algo_identifier)
result = []
for adapter_declaration in chain_adapters:
adapter_instance = self.build_adapter_from_declaration(adapter_declaration)
all_portlet_defined_params = self.reader.get_inputs(self.algo_identifier)
specific_adapter_overwrites = [entry for entry in all_portlet_defined_params
if ATT_OVERWRITE in entry and entry[ATT_OVERWRITE] ==
adapter_declaration[ABCAdapter.KEY_NAME]]
alg_inputs = adapter_instance.get_input_tree()
replace_values = self._prepare_input_tree(alg_inputs, specific_adapter_overwrites)
adapter_configuration = AdapterConfiguration(replace_values, adapter_instance.stored_adapter)
result.append(adapter_configuration)
return result
@classmethod
def build_adapter_from_declaration(cls, adapter_declaration):
"""
Build and adapter from the declaration in the portlets xml.
"""
adapter_import_path = adapter_declaration[ABCAdapter.KEY_TYPE]
class_name = adapter_import_path.split('.')[-1]
module = adapter_import_path.replace('.' + class_name, '')
algo = dao.get_algorithm_by_module(module, class_name)
if algo is not None:
return ABCAdapter.build_adapter(algo)
else:
return None
def _prepare_input_tree(self, input_list, default_values, prefix=''):
"""
Replace the default values from the portlet interface in the adapters
interfaces.
:param input_list: the adapter input tree
:param default_values: the dictionary of overwrites declared in the
portlet xml
:param prefix: in case of a group adapter, the prefix to be added to
each name for the selected subalgorithm
"""
for param in input_list:
for one_value in default_values:
if one_value[ABCAdapter.KEY_NAME] == prefix + param[ABCAdapter.KEY_NAME]:
param[ABCAdapter.KEY_DEFAULT] = one_value[ABCAdapter.KEY_DEFAULT]
if one_value[ABCAdapter.KEY_TYPE] == KEY_DYNAMIC:
## For now just display all dynamic parameters as being disabled.
## If at some point we would like user to be able to select dynamic entry
## should treat this case differently.
param[ABCAdapter.KEY_DISABLED] = True
param[KEY_DYNAMIC] = True
if param.get(ABCAdapter.KEY_OPTIONS) is not None:
new_prefix = prefix + param[ABCAdapter.KEY_NAME] + ABCAdapter.KEYWORD_PARAMS
self._prepare_input_tree(param[ABCAdapter.KEY_OPTIONS], default_values, new_prefix)
if param.get(ABCAdapter.KEY_ATTRIBUTES) is not None:
new_prefix = prefix
if param.get(ABCAdapter.KEY_TYPE) == 'dict':
new_prefix = prefix + param[ABCAdapter.KEY_NAME] + ABCAdapter.KEYWORD_PARAMS
self._prepare_input_tree(param[ABCAdapter.KEY_ATTRIBUTES], default_values, new_prefix)
return input_list
@staticmethod
def update_default_values(portlet_interface, portlet_configuration):
"""
:param portlet_interface: a list of AdapterConfiguration entities.
:param portlet_configuration: a PortletConfiguration entity.
Update the defaults from each AdapterConfiguration entity with the
values stored in the corresponding workflow step held in the
PortletConfiguration entity.
"""
# Check for any defaults first in analyzer steps
if portlet_configuration.analyzers:
for adapter_idx in xrange(len(portlet_interface[:-1])):
saved_configuration = portlet_configuration.analyzers[adapter_idx]
replaced_defaults_dict = InputTreeManager.fill_defaults(portlet_interface[adapter_idx].interface,
saved_configuration.static_param)
portlet_interface[adapter_idx].interface = replaced_defaults_dict
# Check for visualization defaults
if portlet_configuration.visualizer:
saved_configuration = portlet_configuration.visualizer
replaced_defaults_dict = InputTreeManager.fill_defaults(portlet_interface[-1].interface,
saved_configuration.static_param)
portlet_interface[-1].interface = replaced_defaults_dict
def _portlet_dynamic2workflow_step(self, value):
"""
Given a value in portlet specific declaration, eg: step_0[0], return a
dictionary as expected from a workflow step:
{'step_idx' : 0, 'datatype_idx' : 0}
"""
step_idx, datatype_idx = value.replace('step_', '').replace(']', '').split('[')
try:
datatype_idx = int(datatype_idx)
self.log.debug("%s defines an output as an entry to a workflow step." % (value,))
except ValueError, _:
self.log.debug("%s defines an input as an entry to a workflow step." % (value,))
workflow_value = {wf_cfg.STEP_INDEX_KEY: int(step_idx), wf_cfg.DATATYPE_INDEX_KEY: datatype_idx}
return workflow_value
class PortletConfigurer(object):
"""
Helper class that handles all the functionality required from a portlet.
Given a portlet entity, this will allow the following:
- return a configurable interface in the form of a dictionary
equivalent to the input tree of an adapter
- create a new PortletConfiguration entity, or update a already
created one with a new set of parameters
"""
def __init__(self, portlet_entity):
self.log = get_logger(self.__class__.__module__)
self.reader = XMLPortletReader(portlet_entity.xml_path)
self.portlet_entity = portlet_entity
@property
def portlet_id(self):
"""Portlet DB identifier"""
return self.portlet_entity.id
@property
def algo_identifier(self):
"""Unique identifier for current portlet."""
return self.portlet_entity.algorithm_identifier
@property
def ui_name(self):
""" Portlet name to be displayed in UI"""
return self.portlet_entity.name
def get_configurable_interface(self):
"""
Given an algorithm identifier, go trough the adapter chain, and merge
their input tree with the declared overwrites
"""
chain_adapters = self.reader.get_adapters_chain(self.algo_identifier)
result = []
for adapter_declaration in chain_adapters:
adapter_instance = self.build_adapter_from_declaration(
adapter_declaration)
all_portlet_defined_params = self.reader.get_inputs(
self.algo_identifier)
specific_adapter_overwrites = [
entry for entry in all_portlet_defined_params
if ATT_OVERWRITE in entry and entry[ATT_OVERWRITE] ==
adapter_declaration[ABCAdapter.KEY_NAME]
]
alg_inputs = adapter_instance.get_input_tree()
replace_values = self._prepare_input_tree(
alg_inputs, specific_adapter_overwrites)
adapter_configuration = AdapterConfiguration(
replace_values, adapter_instance.stored_adapter)
result.append(adapter_configuration)
return result
@classmethod
def build_adapter_from_declaration(cls, adapter_declaration):
"""
Build and adapter from the declaration in the portlets xml.
"""
adapter_import_path = adapter_declaration[ABCAdapter.KEY_TYPE]
class_name = adapter_import_path.split('.')[-1]
module = adapter_import_path.replace('.' + class_name, '')
algo = dao.get_algorithm_by_module(module, class_name)
if algo is not None:
return ABCAdapter.build_adapter(algo)
else:
return None
def _prepare_input_tree(self, input_list, default_values, prefix=''):
"""
Replace the default values from the portlet interface in the adapters
interfaces.
:param input_list: the adapter input tree
:param default_values: the dictionary of overwrites declared in the
portlet xml
:param prefix: in case of a group adapter, the prefix to be added to
each name for the selected subalgorithm
"""
for param in input_list:
for one_value in default_values:
if one_value[ABCAdapter.
KEY_NAME] == prefix + param[ABCAdapter.KEY_NAME]:
param[ABCAdapter.KEY_DEFAULT] = one_value[
ABCAdapter.KEY_DEFAULT]
if one_value[ABCAdapter.KEY_TYPE] == KEY_DYNAMIC:
## For now just display all dynamic parameters as being disabled.
## If at some point we would like user to be able to select dynamic entry
## should treat this case differently.
param[ABCAdapter.KEY_DISABLED] = True
param[KEY_DYNAMIC] = True
if param.get(ABCAdapter.KEY_OPTIONS) is not None:
new_prefix = prefix + param[
ABCAdapter.KEY_NAME] + ABCAdapter.KEYWORD_PARAMS
self._prepare_input_tree(param[ABCAdapter.KEY_OPTIONS],
default_values, new_prefix)
if param.get(ABCAdapter.KEY_ATTRIBUTES) is not None:
new_prefix = prefix
if param.get(ABCAdapter.KEY_TYPE) == 'dict':
new_prefix = prefix + param[
ABCAdapter.KEY_NAME] + ABCAdapter.KEYWORD_PARAMS
self._prepare_input_tree(param[ABCAdapter.KEY_ATTRIBUTES],
default_values, new_prefix)
return input_list
@staticmethod
def update_default_values(portlet_interface, portlet_configuration):
"""
:param portlet_interface: a list of AdapterConfiguration entities.
:param portlet_configuration: a PortletConfiguration entity.
Update the defaults from each AdapterConfiguration entity with the
values stored in the corresponding workflow step held in the
PortletConfiguration entity.
"""
# Check for any defaults first in analyzer steps
if portlet_configuration.analyzers:
for adapter_idx in xrange(len(portlet_interface[:-1])):
saved_configuration = portlet_configuration.analyzers[
adapter_idx]
replaced_defaults_dict = InputTreeManager.fill_defaults(
portlet_interface[adapter_idx].interface,
saved_configuration.static_param)
portlet_interface[
adapter_idx].interface = replaced_defaults_dict
# Check for visualization defaults
if portlet_configuration.visualizer:
saved_configuration = portlet_configuration.visualizer
replaced_defaults_dict = InputTreeManager.fill_defaults(
portlet_interface[-1].interface,
saved_configuration.static_param)
portlet_interface[-1].interface = replaced_defaults_dict
def _portlet_dynamic2workflow_step(self, value):
"""
Given a value in portlet specific declaration, eg: step_0[0], return a
dictionary as expected from a workflow step:
{'step_idx' : 0, 'datatype_idx' : 0}
"""
step_idx, datatype_idx = value.replace('step_',
'').replace(']', '').split('[')
try:
datatype_idx = int(datatype_idx)
self.log.debug(
"%s defines an output as an entry to a workflow step." %
(value, ))
except ValueError, _:
self.log.debug(
"%s defines an input as an entry to a workflow step." %
(value, ))
workflow_value = {
wf_cfg.STEP_INDEX_KEY: int(step_idx),
wf_cfg.DATATYPE_INDEX_KEY: datatype_idx
}
return workflow_value
