def __init__(self,
processor_id,
component_id,
address=None,
params=None,
enable_proxy=False):
self.processor_id = processor_id
self.component_id = component_id
self._address = address
self.params = params
address = self._address
config = Config()
if address is None:
discovery = _discovery.Discovery(config)
address = discovery.discover_processor_service(processor_id, 'v1')
enable_proxy = enable_proxy or config.get('grpc.enable_proxy', False)
self._channel = grpc.insecure_channel(
address,
options=[("grpc.lb_policy_name", "round_robin"),
("grpc.enable_http_proxy", enable_proxy),
('grpc.max_send_message_length',
config.get('grpc.max_send_message_length')),
('grpc.max_receive_message_length',
config.get('grpc.max_receive_message_length'))])
self._stub = processing_pb2_grpc.ProcessorStub(self._channel)
def test_load_config():
with set_env('MTAP_CONFIG',
str(Path(__file__).parent / 'workingConfig.yaml')):
Config._global_instance = None
c = Config()
assert c['foo'] == 'bar'
assert c['baz.bot'] == [1, 2, 3]
def __init__(self, pipeline, source, params, progress, total, close_events,
max_failures, n_threads, read_ahead, mp_context):
self.pipeline = pipeline
self.failures = 0
self.max_failures = max_failures
self.targets_cond = Condition(Lock())
self.active_targets = 0
self.close_events = close_events
if read_ahead is None:
read_ahead = n_threads
self.max_targets = n_threads + read_ahead
self.n_threads = n_threads
total = (source.total if hasattr(source, 'total') else None) or total
self.progress_bar = tqdm(total=total, unit='event',
smoothing=0.01) if progress else None
if not isinstance(source, ProcessingSource):
if not hasattr(source, '__iter__'):
raise ValueError(
'The source needs to either be a ProcessingSource or an Iterable.'
)
source = _IterableProcessingSource(source)
self.source = source
self.params = params
self.pool = mp_context.Pool(self.n_threads,
initializer=_mp_process_init,
initargs=(Config(), pipeline))
self.active_events = {}
def test_update_from_yaml():
Config._global_instance = None
c = Config()
c.update_from_yaml(str(Path(__file__).parent) + '/workingConfig.yaml')
assert c['foo'] == 'bar'
assert c['baz.bot'] == [1, 2, 3]
def test_enter_twice():
Config._global_instance = None
with Config():
with pytest.raises(ValueError):
with Config():
pass
def test_config_context():
Config._global_instance = None
with Config() as c1:
c1['foo'] = 'bar'
c2 = Config()
assert c2['foo'] == 'bar'
def test_load_broken_config():
Config._global_instance = None
with set_env('MTAP_CONFIG',
str(Path(__file__).parent / 'brokenConfig.yaml')):
with pytest.raises(TypeError):
Config()
def test_load_config():
os.environ['MTAP_CONFIG'] = str(Path(__file__).parent / 'mtapConfig.yaml')
Config._global_instance = None
c = Config()
assert c['foo'] == 'bar'
assert c['baz.bot'] == [1, 2, 3]
def test_load_broken_config():
Config._global_instance = None
os.environ['MTAP_CONFIG'] = str(Path(__file__).parent / 'brokenConfig.yaml')
with pytest.raises(TypeError):
Config()