def test_workers_registry_load_workers_queues():
''' load workers to registry with queues
'''
queues_config = [
{
'name': 'SourceQueue',
'type': 'eventsflow.queues.local.EventsQueue',
},
{
'name': 'TargetQueue',
'type': 'eventsflow.queues.local.EventsQueue',
},
]
queues = QueuesRegistry()
queues.load(queues_config)
workers_config = [
{
'name': 'TestWorker',
'type': 'eventsflow.workers.process.ProcessingWorker',
'inputs': 'SourceQueue',
'outputs': 'TargetQueue',
},
]
registry = WorkersRegistry(queues=queues)
registry.load(workers_config)
assert [type(w) for w in registry.workers()] == [
ProcessingWorker,
]
def test_workers_registry_load_incorrect_workers_config():
''' incorrect workers config, shall be the list of workers, passed as dict
'''
registry = WorkersRegistry(queues=QueuesRegistry())
with pytest.raises(TypeError):
registry.load({
'name': 'TestWorker',
})
def test_workers_registry_load_workers_config_wo_worker_type():
''' incorrect workers config, missed worker type
'''
registry = WorkersRegistry(queues=QueuesRegistry())
with pytest.raises(TypeError):
registry.load([
{
'name': 'TestWorker',
},
])
def test_workers_registry_load_workers_config():
''' correct load of workers config to registry
'''
registry = WorkersRegistry(queues=QueuesRegistry())
registry.load([
{
'name': 'TestWorker',
'type': 'eventsflow.workers.process.ProcessingWorker',
},
])
assert [type(w) for w in registry.workers()] == [
ProcessingWorker,
]
def create_workers_registry(queues: QueuesRegistry) -> WorkersRegistry:
''' return workers registry
'''
registry = WorkersRegistry(queues=queues)
registry.load([
{
'name': 'TestWorker',
'type': 'common_libs.SampleProcessingWorker',
'parameters': {
'timeout': 1,
},
'inputs': 'SourceQueue',
'outputs': 'TargetQueue',
},
])
return registry
def test_workers_registry_flow_worker_status():
''' check workers status
'''
registry = WorkersRegistry(queues=QueuesRegistry())
registry.load([
{
'name': 'TestWorker',
'type': 'tests.common_libs.SampleProcessingWorker',
},
])
assert [ type(w) for w in registry.workers(status='active')] == []
assert [ type(w) for w in registry.workers()] == [ SampleProcessingWorker, ]
assert [ type(w) for w in registry.workers(status='all')] == [ SampleProcessingWorker, ]
assert [ type(w) for w in registry.workers(status='inactive')] == [ SampleProcessingWorker, ]
with pytest.raises(TypeError):
assert [ type(w) for w in registry.workers(status='unknown')] == []
def test_workers_registry_load_workers_config_incorrect_worker_type():
''' incorrect worker type
'''
registry = WorkersRegistry(queues=QueuesRegistry())
with pytest.raises(TypeError):
registry.load([
{
'name': 'TestWorker',
'type': 'eventsflow.workers.ProcessingWorker',
},
])
with pytest.raises(TypeError):
registry.load([
{
'name': 'TestWorker',
'type': 'eventsflow.workers.v2.ProcessingWorker',
},
])
def test_process_worker_stop_processing_by_event():
''' test for Processing Worker, stop processing bt StopProcessing event
'''
queues = QueuesRegistry()
queues.load([
{'name': 'SourceQueue', 'type': 'eventsflow.queues.local.EventsQueue', },
])
workers = WorkersRegistry(queues=queues)
workers.load([
{ 'name': 'TestWorker',
'type': 'common_libs.SampleProcessingWorker',
'parameters': { 'timeout': 1, },
'inputs': 'SourceQueue',
},
])
queues.get('SourceQueue').publish(EventStopProcessing())
for worker in workers.workers():
assert worker.consume() is None
def test_workers_registry_flow_start_workers_and_check_status():
''' run workers and check workers status
'''
queues_registry = QueuesRegistry()
queues_registry.load([
{
'name': 'records',
'type': 'eventsflow.queues.local.EventsQueue',
}
])
workers_registry = WorkersRegistry(queues=queues_registry)
workers_registry.load([
{
'name': 'TestWorker',
'type': 'tests.common_libs.SampleProcessingWorker',
'inputs': 'records'
},
])
assert [ type(w) for w in workers_registry.workers(status='active')] == [ ]
workers_registry.start()
assert [
type(w) for w in workers_registry.workers(status='active')
] == [ SampleProcessingWorker, ]
queues_registry.get('records').publish(EventStopProcessing())
time.sleep(1)
assert [
type(w) for w in workers_registry.workers(status='active')
] == []
assert [
type(w) for w in workers_registry.workers(status='inactive')
] == [ SampleProcessingWorker, ]
def test_process_worker_run():
''' test for Processing Worker run
'''
test_events = [
{'name': 'EventTest#1', 'metadata': {}, 'payload': []},
]
queues = create_queues_registry()
workers = WorkersRegistry(queues=queues)
workers.load([
{ 'name': 'TestWorker',
'type': 'common_libs.SampleProcessingWorker',
'parameters': { 'timeout': 1, },
'inputs': [
{'name': 'default', 'refs': 'SourceQueue', 'events': test_events }
],
'outputs': [
{'name': 'default', 'refs': 'TargetQueue', }
],
},
])
for worker in workers.workers():
worker.run()
source_queue = queues.get('SourceQueue')
assert source_queue
assert source_queue.empty() is True
target_queue = queues.get('TargetQueue')
assert target_queue
for event in test_events:
event['metadata']['eventsflow.source'] = 'TestWorker#000'
assert event == target_queue.consume().to_dict()
target_queue.commit()
def test_workers_registry_load_empty_workers_config():
''' test load empty workers config
'''
registry = WorkersRegistry(queues=QueuesRegistry())
registry.load([])
assert list(registry.workers(status='all')) == list()
def test_workers_registry_load_workers_queues_with_events():
''' load workers to registry with queues and events
'''
queues_config = [
{
'name': 'SourceQueue',
'type': 'eventsflow.queues.local.EventsQueue',
},
{
'name': 'TargetQueue',
'type': 'eventsflow.queues.local.EventsQueue',
},
]
queues = QueuesRegistry()
queues.load(queues_config)
test_events = [
{
'name': 'EventTest#1',
'metadata': {},
'payload': []
},
{
'name': 'EventTest#1',
'metadata': {},
'payload': []
},
{
'name': 'EventTest#1',
'metadata': {},
'payload': []
},
]
workers_config = [
{
'name':
'TestWorker',
'type':
'eventsflow.workers.process.ProcessingWorker',
'inputs': [{
'name': 'default',
'refs': 'SourceQueue',
'events': test_events
}],
'outputs': [{
'name': 'default',
'refs': 'TargetQueue',
'events': test_events
}],
},
]
registry = WorkersRegistry(queues=queues)
registry.load(workers_config)
assert [type(w) for w in registry.workers()] == [
ProcessingWorker,
]
assert queues.get('SourceQueue')
assert queues.get('TargetQueue')
# Source Queue
events = []
for _ in test_events:
event = queues.get('SourceQueue').consume()
events.append(event.to_dict())
assert events == test_events
# Target Queue
events = []
for _ in test_events:
event = queues.get('TargetQueue').consume()
events.append(event.to_dict())
assert events == test_events
class Flow:
''' Processing Flow
'''
def __init__(self, path:str, extra_vars=None):
''' Initialize Processing Flow
### Parameters
- path: the path to a flow configuration file
- extra_vars: extra variables, optional. From command line `--vars` argument
'''
if not extra_vars:
extra_vars = list()
queues, workers = FlowParser(path).parse(extra_vars)
self._queues = QueuesRegistry()
self._queues.load(queues)
self._workers = WorkersRegistry(queues=self._queues)
self._workers.load(workers)
@property
def queues(self) -> List:
''' returns the list of queues
'''
return self._queues.queues
@property
def workers(self) -> List:
''' returns the list of workers
'''
return self._workers.workers
def get_current_status(self, with_logging:bool=False):
''' returns the current workers status
'''
active_workers = list(self._workers.workers(status='active'))
stopped_workers = list(self._workers.workers(status='inactive'))
current_state = {
'activeWorkers': active_workers,
'activeWorkersByName': [p.name for p in active_workers],
'inactiveWorkersByName': [p.name for p in stopped_workers],
'queues': dict([
(name, queue.size()) for name, queue in self._queues.queues.items()
]),
}
if with_logging:
logger.info('Queues stats: %s', current_state.get('queues', []))
logger.info('Active workers: %s', current_state.get('activeWorkersByName', []))
logger.info('Stopped workers: %s', current_state.get('stoppedWorkersByName', []))
return current_state
def run(self):
''' run flow
'''
# start all workers in the registry
self._workers.start()
while True:
current_status = self.get_current_status(with_logging=True)
if not current_status.get('activeWorkers', []):
break
for queue_name, queue_instance in self._queues.queues.items():
queue_size = queue_instance.size()
num_producers = 0
for producer in current_status.get('activeWorkers', []):
for output_queue_instance in getattr(producer, 'outputs', dict()).values():
if output_queue_instance and output_queue_instance == queue_instance:
num_producers += 1
num_consumers = 0
for consumer in current_status.get('activeWorkers', []):
for input_queue_instance in getattr(consumer, 'inputs', dict()).values():
if input_queue_instance and input_queue_instance == queue_instance:
num_consumers += 1
if queue_size == 0 and num_producers == 0:
for _ in range(num_consumers):
logger.info('Sending stop processing task to queue: %s', queue_name)
queue_instance.put(EventStopProcessing)
time.sleep(FlowSettings.STATUS_CHECK_TIME_INTERVAL)
logger.info('Processing completed')