def init_sequential(self):
"""
Initialise Flow for sequential mode
Returns
-------
True if every initialisation are correct
Otherwise False
"""
self.configure_ports()
self.sequential_instances = dict()
# set coroutines
#producer
conf = self.get_step_conf(self.producer_step.name)
module = conf['module']
class_name = conf['class']
try:
coroutine = dynamic_class_from_module(class_name, module, self)
except DynamicClassError as e:
self.log.error('{}'.format(e))
return False
self.producer = ProducerSequential(coroutine, name=self.producer_step.name,
connections=self.producer_step.connections,
main_connection_name = self.producer_step.main_connection_name)
self.producer.init()
self.producer_step.process.append(self.producer)
self.sequential_instances[self.producer_step.name] = self.producer
#stages
for step in (self.stager_steps ):
conf = self.get_step_conf(step.name)
module = conf['module']
class_name = conf['class']
try:
coroutine = dynamic_class_from_module(class_name, module, self)
except DynamicClassError as e:
self.log.error('{}'.format(e))
return False
stage = StagerSequential(coroutine,name = step.name, connections=step.connections,
main_connection_name=step.main_connection_name)
step.process.append(stage)
self.sequential_instances[step.name] = stage
self.stagers.append(stage)
stage.init()
#consumer
conf = self.get_step_conf(self.consumer_step.name)
module = conf['module']
class_name = conf['class']
try:
coroutine = dynamic_class_from_module(class_name, module, self)
except DynamicClassError as e:
self.log.error('{}'.format(e))
return False
self.consumer = ConsumerSequential(coroutine, name = conf['name'])
self.consumer_step.process.append(self.consumer)
self.consumer.init()
self.sequential_instances[self.consumer_step.name] = self.consumer
self.display_conf()
return True
def init_sequential(self):
"""
Initialise Flow for sequential mode
Returns:
========
True if every initialisation are correct
Otherwise False
"""
self.configure_ports()
self.sequential_instances = dict()
# set coroutines
#producer
conf = self.get_step_conf(self.producer_step.name)
module = conf['module']
class_name = conf['class']
try:
coroutine = dynamic_class_from_module(class_name, module, self)
except DynamicClassError as e:
self.log.error('{}'.format(e))
return False
self.producer = ProducerSequential(coroutine, name=self.producer_step.name,
connexions=self.producer_step.connexions,
main_connexion_name = self.producer_step.main_connexion_name)
self.producer.init()
self.producer_step.processus.append(self.producer)
self.sequential_instances[self.producer_step.name] = self.producer
#stages
for step in (self.stager_steps ):
conf = self.get_step_conf(step.name)
module = conf['module']
class_name = conf['class']
try:
coroutine = dynamic_class_from_module(class_name, module, self)
except DynamicClassError as e:
self.log.error('{}'.format(e))
return False
stage = StagerSequential(coroutine,name = step.name, connexions=step.connexions,
main_connexion_name=step.main_connexion_name)
step.processus.append(stage)
self.sequential_instances[step.name] = stage
self.stagers.append(stage)
stage.init()
#consumer
conf = self.get_step_conf(self.consumer_step.name)
module = conf['module']
class_name = conf['class']
try:
coroutine = dynamic_class_from_module(class_name, module, self)
except DynamicClassError as e:
self.log.error('{}'.format(e))
return False
self.consumer = ConsumerSequential(coroutine, name = conf['name'])
self.consumer_step.processus.append(self.consumer)
self.consumer.init()
self.sequential_instances[self.consumer_step.name] = self.consumer
self.display_conf()
return True
def instantiation(self,
name,
stage_type,
process_name=None,
port_in=None,
connections=None,
main_connection_name=None,
config=None):
'''
Instantiate on Python object from name found in configuration
Parameters
----------
name : str
stage name
stage_type : str
process_name : str
port_in : str
step ZMQ port in
connections : dict
key: StepName, value" connection ZMQ ports
main_connection_name : str
main ZMQ connection name. Connexion to use when user not precise
'''
stage = self.get_step_conf(name)
module = stage['module']
class_name = stage['class']
obj = dynamic_class_from_module(class_name, module, self)
if obj is None:
raise FlowError('Cannot create instance of ' + name)
obj.name = name
if stage_type == self.STAGER:
process = StagerZmq(obj,
port_in,
process_name,
connections=connections,
main_connection_name=main_connection_name)
elif stage_type == self.PRODUCER:
process = ProducerZmq(obj,
name,
connections=connections,
main_connection_name=main_connection_name)
elif stage_type == self.CONSUMER:
process = ConsumerZMQ(obj, port_in, name)
else:
raise FlowError('Cannot create instance of', name, '. Type',
stage_type, 'does not exist.')
# set coroutine socket to it's stager or producer socket .
return process
def instantiation(self, name, stage_type, process_name=None, port_in=None,
connections=None, main_connection_name=None, config=None):
'''
Instantiate on Python object from name found in configuration
Parameters
----------
name : str
stage name
stage_type : str
process_name : str
port_in : str
step ZMQ port in
connections : dict
key: StepName, value" connection ZMQ ports
main_connection_name : str
main ZMQ connection name. Connexion to use when user not precise
'''
stage = self.get_step_conf(name)
module = stage['module']
class_name = stage['class']
obj = dynamic_class_from_module(class_name, module, self)
if obj is None:
raise FlowError('Cannot create instance of ' + name)
obj.name = name
if stage_type == self.STAGER:
process = StagerZmq(
obj, port_in, process_name,
connections=connections,
main_connection_name = main_connection_name)
elif stage_type == self.PRODUCER:
process = ProducerZmq(
obj, name, connections=connections,
main_connection_name= main_connection_name)
elif stage_type == self.CONSUMER:
process = ConsumerZMQ(
obj,port_in,
name)
else:
raise FlowError(
'Cannot create instance of', name, '. Type',
stage_type, 'does not exist.')
# set coroutine socket to it's stager or producer socket .
return process
def instantiation(
self, section_name, stage_type, port_in=None,
port_out=None, config=None, name=''):
'''
Instantiate on Pytohn object from name found in configuration
Parameters
----------
section_name : str
section name in configuration file
stage_type : str
port_in : str
step port in
port_out: str
step port out
name : str
stage name
'''
stage = self.get_step_conf(section_name)
module = stage['module']
class_name = stage['class']
obj = dynamic_class_from_module(class_name, module, self)
if obj is None:
raise PipelineError('Cannot create instance of ' + section_name)
obj.section_name = section_name
if stage_type == self.STAGER:
thread = StagerZmq(
obj, port_in, port_out, name=name, gui_address=self.gui_address)
elif stage_type == self.PRODUCER:
thread = ProducerZmq(
obj, port_out, 'producer', gui_address=self.gui_address)
elif stage_type == self.CONSUMER:
thread = ConsumerZMQ(
obj, port_in,
'consumer', parent=self,
gui_address=self.gui_address)
else:
raise PipelineError(
'Cannot create instance of', section_name, '. Type',
stage_type, 'does not exist.')
# set coroutine socket to it's stager or producer socket .
return thread
