def test_workers_registry_load_workers_queues():
''' load workers to registry with queues
'''
queues_config = [
{
'name': 'SourceQueue',
'type': 'eventsflow.queues.local.EventsQueue',
},
{
'name': 'TargetQueue',
'type': 'eventsflow.queues.local.EventsQueue',
},
]
queues = QueuesRegistry()
queues.load(queues_config)
workers_config = [
{
'name': 'TestWorker',
'type': 'eventsflow.workers.process.ProcessingWorker',
'inputs': 'SourceQueue',
'outputs': 'TargetQueue',
},
]
registry = WorkersRegistry(queues=queues)
registry.load(workers_config)
assert [type(w) for w in registry.workers()] == [
ProcessingWorker,
]
def test_workers_registry_load_incorrect_workers_config():
''' incorrect workers config, shall be the list of workers, passed as dict
'''
registry = WorkersRegistry(queues=QueuesRegistry())
with pytest.raises(TypeError):
registry.load({
'name': 'TestWorker',
})
def test_queues_registry_load_incorrect_queues_config():
''' test for load incorrect queues config
'''
registry = QueuesRegistry()
with pytest.raises(TypeError):
registry.load({
'name': 'TestQueue#1',
'type': 'eventsflow.queues.local.EventsQueue',
})
def test_workers_registry_load_workers_config_wo_worker_type():
''' incorrect workers config, missed worker type
'''
registry = WorkersRegistry(queues=QueuesRegistry())
with pytest.raises(TypeError):
registry.load([
{
'name': 'TestWorker',
},
])
def test_queues_registry_load_queues_config_with_missed_queue_name():
''' test for loading queues with missed queue name
'''
registry = QueuesRegistry()
with pytest.raises(TypeError):
registry.load([
{
'type': 'eventsflow.queues.local.EventsQueue',
},
])
def test_queues_registry_load_queues_config_wo_queue_type():
''' test for loading queues without queue type
'''
registry = QueuesRegistry()
with pytest.raises(TypeError):
registry.load([
{
'name': 'TestQueue#1',
},
])
def test_queues_registry_load_queues_config_with_unk_queue_type():
''' tests for loading queue with unknown queue type
'''
registry = QueuesRegistry()
with pytest.raises(TypeError):
registry.load([
{
'name': 'TestQueue#1',
'type': 'eventsflow.queues.EventsQueue',
},
])
def test_workers_registry_load_workers_config():
''' correct load of workers config to registry
'''
registry = WorkersRegistry(queues=QueuesRegistry())
registry.load([
{
'name': 'TestWorker',
'type': 'eventsflow.workers.process.ProcessingWorker',
},
])
assert [type(w) for w in registry.workers()] == [
ProcessingWorker,
]
def create_queues_registry() -> QueuesRegistry:
''' return queues registry with source and target queues
'''
registry = QueuesRegistry()
registry.load([
{
'name': 'SourceQueue',
'type': 'eventsflow.queues.local.EventsQueue',
},
{
'name': 'TargetQueue',
'type': 'eventsflow.queues.local.EventsQueue',
},
])
return registry
def test_queues_registry_load():
''' load queues to registry
'''
registry = QueuesRegistry()
registry.load([
{
'name': 'TestQueue#1',
'type': 'eventsflow.queues.local.EventsQueue',
},
])
assert registry is not None
assert registry.get('TestQueue#1') is not None
assert isinstance(registry.get('TestQueue#1'), EventsQueue)
assert list(registry.queues) == [
'TestQueue#1',
]
def test_workers_registry_flow_worker_status():
''' check workers status
'''
registry = WorkersRegistry(queues=QueuesRegistry())
registry.load([
{
'name': 'TestWorker',
'type': 'tests.common_libs.SampleProcessingWorker',
},
])
assert [ type(w) for w in registry.workers(status='active')] == []
assert [ type(w) for w in registry.workers()] == [ SampleProcessingWorker, ]
assert [ type(w) for w in registry.workers(status='all')] == [ SampleProcessingWorker, ]
assert [ type(w) for w in registry.workers(status='inactive')] == [ SampleProcessingWorker, ]
with pytest.raises(TypeError):
assert [ type(w) for w in registry.workers(status='unknown')] == []
def __init__(self, path:str, extra_vars=None):
''' Initialize Processing Flow
### Parameters
- path: the path to a flow configuration file
- extra_vars: extra variables, optional. From command line `--vars` argument
'''
if not extra_vars:
extra_vars = list()
queues, workers = FlowParser(path).parse(extra_vars)
self._queues = QueuesRegistry()
self._queues.load(queues)
self._workers = WorkersRegistry(queues=self._queues)
self._workers.load(workers)
def test_workers_registry_load_workers_config_incorrect_worker_type():
''' incorrect worker type
'''
registry = WorkersRegistry(queues=QueuesRegistry())
with pytest.raises(TypeError):
registry.load([
{
'name': 'TestWorker',
'type': 'eventsflow.workers.ProcessingWorker',
},
])
with pytest.raises(TypeError):
registry.load([
{
'name': 'TestWorker',
'type': 'eventsflow.workers.v2.ProcessingWorker',
},
])
def test_process_worker_stop_processing_by_event():
''' test for Processing Worker, stop processing bt StopProcessing event
'''
queues = QueuesRegistry()
queues.load([
{'name': 'SourceQueue', 'type': 'eventsflow.queues.local.EventsQueue', },
])
workers = WorkersRegistry(queues=queues)
workers.load([
{ 'name': 'TestWorker',
'type': 'common_libs.SampleProcessingWorker',
'parameters': { 'timeout': 1, },
'inputs': 'SourceQueue',
},
])
queues.get('SourceQueue').publish(EventStopProcessing())
for worker in workers.workers():
assert worker.consume() is None
def test_workers_registry_flow_start_workers_and_check_status():
''' run workers and check workers status
'''
queues_registry = QueuesRegistry()
queues_registry.load([
{
'name': 'records',
'type': 'eventsflow.queues.local.EventsQueue',
}
])
workers_registry = WorkersRegistry(queues=queues_registry)
workers_registry.load([
{
'name': 'TestWorker',
'type': 'tests.common_libs.SampleProcessingWorker',
'inputs': 'records'
},
])
assert [ type(w) for w in workers_registry.workers(status='active')] == [ ]
workers_registry.start()
assert [
type(w) for w in workers_registry.workers(status='active')
] == [ SampleProcessingWorker, ]
queues_registry.get('records').publish(EventStopProcessing())
time.sleep(1)
assert [
type(w) for w in workers_registry.workers(status='active')
] == []
assert [
type(w) for w in workers_registry.workers(status='inactive')
] == [ SampleProcessingWorker, ]
def test_queues_registry_load_empty_queues_config():
''' test for load empty queues
'''
registry = QueuesRegistry()
with pytest.raises(TypeError):
registry.load([])
def test_queues_registry_init():
''' test for Queue Registry Initialization
'''
registry = QueuesRegistry()
assert registry is not None
def test_workers_registry_init():
''' test for worker registry initialization
'''
registry = WorkersRegistry(queues=QueuesRegistry())
assert registry is not None
def test_workers_registry_load_workers_queues_with_events():
''' load workers to registry with queues and events
'''
queues_config = [
{
'name': 'SourceQueue',
'type': 'eventsflow.queues.local.EventsQueue',
},
{
'name': 'TargetQueue',
'type': 'eventsflow.queues.local.EventsQueue',
},
]
queues = QueuesRegistry()
queues.load(queues_config)
test_events = [
{
'name': 'EventTest#1',
'metadata': {},
'payload': []
},
{
'name': 'EventTest#1',
'metadata': {},
'payload': []
},
{
'name': 'EventTest#1',
'metadata': {},
'payload': []
},
]
workers_config = [
{
'name':
'TestWorker',
'type':
'eventsflow.workers.process.ProcessingWorker',
'inputs': [{
'name': 'default',
'refs': 'SourceQueue',
'events': test_events
}],
'outputs': [{
'name': 'default',
'refs': 'TargetQueue',
'events': test_events
}],
},
]
registry = WorkersRegistry(queues=queues)
registry.load(workers_config)
assert [type(w) for w in registry.workers()] == [
ProcessingWorker,
]
assert queues.get('SourceQueue')
assert queues.get('TargetQueue')
# Source Queue
events = []
for _ in test_events:
event = queues.get('SourceQueue').consume()
events.append(event.to_dict())
assert events == test_events
# Target Queue
events = []
for _ in test_events:
event = queues.get('TargetQueue').consume()
events.append(event.to_dict())
assert events == test_events
def test_workers_registry_load_empty_workers_config():
''' test load empty workers config
'''
registry = WorkersRegistry(queues=QueuesRegistry())
registry.load([])
assert list(registry.workers(status='all')) == list()