class ModulesController(object):
""" Base class for a ModulesController.
"""
# is_chief = True
# """
# Whether this process is the chief worker in multi-pipeline.
# Certain Modules will only be run by chief worker.
# """
def __init__(self):
self._modules = []
self.ModuleTable = {}
@call_only_once
def build_pipeline(self, modules):
"""
Setup Modules to built the modules pipeline.
Args:
modules ([Module]): list of Module instances
"""
assert isinstance(modules, list), modules
for md in modules:
self.Check_Module(md)
# some final operations that might modify the pipeline
logger.logger.info("Build Modules pipeline ...")
self._modules = Modules(self._modules)
self._modules.register_to_controller(weakref.proxy(self))
self._modules = self._modules.get_modules()
def run_pipeline(self,modules,inputs):
"""
Implemented by two lines:
.. code-block:: python
self.build_pipeline(modules)
self.main_loop(inputs)
You can call those methods by yourself to have better control on details if needed.
"""
self.build_pipeline(modules)
self.main_loop(inputs)
def run_pipeline_with_defaults(self, modules=None, extra_modules=None,inputs=None):
"""
Same as :meth:`run_pipeline()`, except:
1. Add `extra_modules` to modules. The default value for
`extra_modules` is :meth:`DEFAULT_MODULES()`.
"""
modules = copy.copy(modules or [])
extra_modules = DEFAULT_MODULES() if extra_modules is None else extra_modules
modules.extend(extra_modules)
self.run_pipeline(modules,inputs)
@call_only_once
def main_loop(self, inputs):
"""
Run the main piepline loop.
Args:
inputs (tuple): inputs must be a tuple(thus we can treat multi-input and multi-output)
"""
logger.logger.info("Begin Modules pipeline ...")
for index, cb in enumerate(self._modules):
inputs = self.run_step(inputs, index)
self.outputs = inputs
def run_step(self,inputs,index):
"""
Defines what to do in one stage of pipeline. The default is:
'' run one module ''.
The behavior of each iteration can be changed by overriding this method.
"""
#TODO 如何使得当前模块的输入只用到前一个模块的部分输出？
if index != 0:
current_module_input_desc = self.ModuleTable[
self._modules[index].__class__.__name__].get_inputs_desc()
pre_module_output_desc = self.ModuleTable[
self._modules[index-1].__class__.__name__].get_outputs_desc()
if not current_module_input_desc==pre_module_output_desc:
raise Stop(
'module {} and {} can\'t be pipelined ,it may be caused by inconsistent input and output.'
'check description of those modules for details.'
.format(self._modules[index-1].__class__.__name__,self._modules[index].__class__.__name__))
if not type(inputs)==type(tuple):
inputs = (inputs,)
return self._modules[index].run(inputs)
def _Check_Module(self, md):
"""
Check Modules which are passed to ModulesController by Config.
It can only be called before :meth:`ModulesController.run_pipeline()`.
Args:
md (Module or [Module]): a Module or a list of Modules
Returns:
succeed or not
"""
if isinstance(md, (list, tuple)):
for x in md:
self._Check_Module(x)
return
assert isinstance(md, Module), md
assert not isinstance(self._modules, Modules), \
"Cannot append more Modules after ModulesController was setup!"
self._modules.append(md)
return True
Check_Module = _Check_Module
def register_ModuleDesc_to_ModuleTable(self,Module_name,desc):
self.ModuleTable[Module_name] = desc
def __new__(cls, *args, **kwargs):
if (len(args) > 0 and isinstance(args[0], Config)) \
or 'config' in kwargs:
logger.logger.error("Use interface API to launch controller ,do not pass Config directly to a controller!")
import sys
sys.exit(1)
else:
return super(ModulesController, cls).__new__(cls)
