def __init__(self,
host: str,
*,
port: int = 0,
register: bool = False,
workers: int = 10,
write_address: bool = False,
config: 'Optional[mtap.Config]' = None):
if host is None:
host = '127.0.0.1'
if port is None:
port = 0
self.write_address = write_address
thread_pool = ThreadPoolExecutor(max_workers=workers)
if config is None:
config = _config.Config()
server = grpc.server(thread_pool,
options=[
('grpc.max_send_message_length',
config['grpc.max_send_message_length']),
('grpc.max_receive_message_length',
config['grpc.max_receive_message_length'])
])
servicer = EventsServicer()
events_pb2_grpc.add_EventsServicer_to_server(servicer, server)
health_servicer = health.HealthServicer()
health_servicer.set('', 'SERVING')
health_servicer.set(constants.EVENTS_SERVICE_NAME, 'SERVING')
health_pb2_grpc.add_HealthServicer_to_server(health_servicer, server)
self._port = server.add_insecure_port(host + ':' + str(port))
self._server = server
self._address = host
self._config = _config.Config()
self._register = register
self._address_file = None
def __init__(self,
address: Optional[Union[str, Iterable[str]]],
channel_factory: Callable[[str], grpc.Channel] = None,
_pool: Optional[_ClientPool] = None,
_channel: Optional[grpc.Channel] = None):
if address is None:
discovery = _discovery.Discovery(_config.Config())
address = discovery.discover_events_service('v1')
if isinstance(address, str):
self.addresses = [address]
elif isinstance(address, Iterable):
self.addresses = list(address)
else:
raise ValueError("Unrecognized address type: " +
str(type(address)))
self.channel_factory = channel_factory
if channel_factory is None:
self.channel_factory = create_channel
self._pool = _pool
if self._pool is None:
self._pool = _ClientPool(self.channel_factory, self.addresses)
self._channel = _channel
if self._channel is not None:
self.stub = events_pb2_grpc.EventsStub(self._channel)
r = self.stub.GetEventsInstanceId(
events_pb2.GetEventsInstanceIdRequest())
self._instance_id = r.instance_id
self._is_open = True
def __init__(self,
proc: 'mtap.EventProcessor',
identifier: str,
workers: 'Optional[int]' = 8,
events_address: 'Optional[Union[str, Sequence[str]]]' = None,
mp_context=None):
if mp_context is None:
import multiprocessing as mp
mp_context = mp
config = _config.Config()
self.pool = mp_context.Pool(workers,
initializer=_mp_initialize,
initargs=(proc, events_address,
dict(config)))
self.metadata = proc.metadata
self.processor_id = identifier
self.component_id = identifier
def create_channel(address):
config = _config.Config()
enable_proxy = config.get('grpc.enable_proxy', False)
channel = grpc.insecure_channel(
address,
options=[('grpc.enable_http_proxy', enable_proxy),
('grpc.max_send_message_length',
config.get('grpc.max_send_message_length')),
('grpc.max_receive_message_length',
config.get('grpc.max_receive_message_length'))])
health = health_pb2_grpc.HealthStub(channel)
hcr = health.Check(
health_pb2.HealthCheckRequest(service=constants.EVENTS_SERVICE_NAME))
if hcr.status != health_pb2.HealthCheckResponse.SERVING:
raise ValueError('Failed to connect to events service. Status:')
return channel
def __init__(self,
runner: 'mtap.processing.ProcessingComponent',
host: str,
port: int = 0,
*,
register: bool = False,
workers: Optional[int] = None,
write_address: bool = False,
config: 'Optional[mtap.Config]' = None):
self.host = host
self._port = port
self.processor_id = runner.processor_id
self.write_address = write_address
if config is None:
config = _config.Config()
self._health_servicer = health.HealthServicer()
self._health_servicer.set('', 'SERVING')
self._servicer = _ProcessorServicer(
config=config,
address=host,
runner=runner,
health_servicer=self._health_servicer,
register=register)
workers = workers or 10
thread_pool = thread.ThreadPoolExecutor(max_workers=workers)
self._server = grpc.server(
thread_pool,
options=[('grpc.max_send_message_length',
config.get('grpc.max_send_message_length')),
('grpc.max_receive_message_length',
config.get('grpc.max_receive_message_length'))])
health_pb2_grpc.add_HealthServicer_to_server(self._health_servicer,
self._server)
processing_pb2_grpc.add_ProcessorServicer_to_server(
self._servicer, self._server)
self._port = self._server.add_insecure_port("{}:{}".format(
self.host, self.port))
self._stopped_event = threading.Event()
self._address_file = None
def run_servers(self):
"""Starts all of the configured services.
Raises:
ServiceDeploymentException: If one of the services fails to launch.
"""
with _config.Config() as c:
enable_proxy = c.get('grpc.enable_proxy', False)
processes_listeners = []
events_addresses = []
def shutdown(kill=False):
print("Shutting down all processors")
for p, listener in processes_listeners:
if kill:
p.kill()
listener.join(timeout=1)
if self.events_deployment.enabled:
for call in self.events_deployment.create_calls(
self.global_settings):
p = subprocess.Popen(call,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
listener, events_address = _listen_test_connectivity(
p, "events", 30, enable_proxy)
events_addresses.append(events_address)
processes_listeners.append((p, listener))
if (not self.global_settings.register
and not self.events_deployment.service_deployment.register
and self.shared_processor_config.events_addresses is None):
self.shared_processor_config.events_addresses = events_addresses
for processor_deployment in self.processors:
if processor_deployment.enabled:
for call in processor_deployment.create_calls(
self.global_settings,
self.shared_processor_config):
logger.debug('Launching processor with call: %s', call)
p = subprocess.Popen(call,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
startup_timeout = (
processor_deployment.startup_timeout
or self.shared_processor_config.startup_timeout)
try:
listener, _ = _listen_test_connectivity(
p, call, startup_timeout, enable_proxy)
except ServiceDeploymentException as e:
logger.error(str(e))
return shutdown(kill=True)
processes_listeners.append((p, listener))
print('Done deploying all servers.', flush=True)
try:
while True:
time.sleep(60 * 60 * 24)
except KeyboardInterrupt:
return shutdown()
def run_processor(proc: 'mtap.EventProcessor',
*,
mp: bool = False,
options: Optional[argparse.Namespace] = None,
args: Optional[Sequence[str]] = None,
mp_context=None):
"""Runs the processor as a GRPC service, blocking until an interrupt signal is received.
Args:
proc (EventProcessor): The processor to host.
mp (bool): If true, will create instances of ``proc`` on multiple forked processes to
process events. This is useful if the processor is computationally intensive and would
run into Python GIL issues on a single process.
options (~typing.Optional[~argparse.Namespace]): The parsed arguments from the parser
returned by :func:`processor_parser`.
args (~typing.Optional[~typing.Sequence[str]]): Arguments to parse server settings from if
``namespace`` was not supplied.
mp_context: A multiprocessing context that gets passed to the process pool executor in the
case of mp = True.
Examples:
Will automatically parse arguments:
>>> run_processor(MyProcessor())
Manual arguments:
>>> run_processor(MyProcessor(), args=['-p', '8080'])
"""
from mtap import EventProcessor
if not isinstance(proc, EventProcessor):
raise ValueError("Processor must be instance of EventProcessor class.")
if options is None:
processors_parser = argparse.ArgumentParser(
parents=[processor_parser()])
processors_parser.add_help = True
options = processors_parser.parse_args(args)
if options.log_level:
logging.basicConfig(level=getattr(logging, options.log_level))
events_addresses = []
if options.events_addresses is not None:
events_addresses.extend(options.events_addresses.split(','))
with _config.Config() as c:
if options.mtap_config is not None:
c.update_from_yaml(options.mtap_config)
# instantiate runner
processor_id = options.identifier or proc.metadata['name']
enable_http_proxy = options.grpc_enable_http_proxy
if enable_http_proxy is not None:
c['grpc.enable_proxy'] = enable_http_proxy
if mp:
runner = MpProcessorRunner(proc=proc,
workers=options.workers,
events_address=events_addresses,
identifier=processor_id,
mp_context=mp_context)
else:
client = data.EventsClient(address=events_addresses)
runner = _runners.ProcessorRunner(proc,
client=client,
identifier=processor_id,
params=None)
server = ProcessorServer(runner=runner,
host=options.host,
port=options.port,
register=options.register,
workers=options.workers,
write_address=options.write_address)
server.start()
try:
while True:
time.sleep(_ONE_DAY_IN_SECONDS)
except KeyboardInterrupt:
server.stop()
def _mp_initialize(proc: 'mtap.EventProcessor', events_address, config):
global _mp_processor
global _mp_client
_config.Config(config)
_mp_processor = proc
_mp_client = data.EventsClient(address=events_address)
