def test_aggregator_multiple_components(self): aggr = Aggregator(_get_metrics_one, _get_metrics_two, name='test') self.assertEqual(list(aggr.get_metrics()), [ ('dummy', 'one', {'name': 'test'}), ('dummy', 'two', {'name': 'test'}), ])
def test_aggregator_add_multiple_tags(self): aggr = Aggregator(_get_metrics_one, name='test') aggr.add_tags(origin='localhost', time='now') self.assertEqual(list(aggr.get_metrics()), [ ('dummy', 'one', {'name': 'test', 'origin': 'localhost', 'time': 'now'}) ])
def test_aggregator_add_multiple_tags(self): aggr = Aggregator([ Gauge('dummy', 'one') ], tags={'name': 'test'}) aggr.add_tags(origin='localhost', time='now') self.assertIn(('dummy', 'one', {'name': 'test', 'origin': 'localhost', 'time': 'now'}), list(aggr))
def __init__(self, rpc=None, registry=None, events=None, log_endpoint=None, service_name=None, debug=False, monitor_endpoint=None, pool=None): if pool is None: pool = trace.Group() super(ServiceContainer, self).__init__(error_hook=Hook('error_hook'), pool=pool) self.log_endpoint = log_endpoint self.backdoor_endpoint = None self.service_name = service_name self.fqdn = socket.getfqdn() self.http_request_hook = Hook('http_request_hook') self.server = rpc self.service_registry = registry self.events = events self.installed_interfaces = {} self.installed_plugins = [] self.debug = debug self.monitor_endpoint = monitor_endpoint self.metrics_aggregator = Aggregator(self._get_metrics, service=self.service_name, host=self.fqdn) if self.service_registry: self.add_component(self.service_registry) if self.events: self.add_component(self.events) self.events.install(self) self.monitor = self.install(MonitorPusher, aggregator=self.metrics_aggregator, endpoint=self.monitor_endpoint, interval=5) self.add_component(rpc) rpc.request_handler = self.handle_request self.install_interface(DefaultInterface, name='lymph')
def __init__(self, ip='127.0.0.1', port=None, registry=None, events=None, node_endpoint=None, log_endpoint=None, service_name=None, debug=False, monitor_endpoint=None, pool_size=None): super(ServiceContainer, self).__init__() self.node_endpoint = node_endpoint self.log_endpoint = log_endpoint self.backdoor_endpoint = None self.service_name = service_name self.fqdn = socket.getfqdn() self.service_registry = registry self.event_system = events self.error_hook = Hook() self.pool = trace.Group(size=pool_size) self.installed_interfaces = {} self.installed_plugins = [] self.debug = debug self.monitor_endpoint = monitor_endpoint self.metrics_aggregator = Aggregator(self._get_metrics, service=self.service_name, host=self.fqdn) if self.service_registry: self.add_component(self.service_registry) self.service_registry.install(self) if self.event_system: self.add_component(self.event_system) self.event_system.install(self) self.monitor = self.install(MonitorPusher, aggregator=self.metrics_aggregator, endpoint=self.monitor_endpoint, interval=5) self.server = self.install(self.server_cls, ip=ip, port=port) self.install_interface(DefaultInterface, name='lymph')
def __init__(self, rpc=None, registry=None, events=None, log_endpoint=None, service_name=None, debug=False, monitor_endpoint=None, pool=None): if pool is None: pool = trace.Group() super(ServiceContainer, self).__init__(error_hook=Hook('error_hook'), pool=pool) self.log_endpoint = log_endpoint self.backdoor_endpoint = None self.service_name = service_name self.fqdn = socket.getfqdn() self.server = rpc self.service_registry = registry self.event_system = events self.installed_interfaces = {} self.installed_plugins = [] self.debug = debug self.monitor_endpoint = monitor_endpoint self.metrics_aggregator = Aggregator(self._get_metrics, service=self.service_name, host=self.fqdn) if self.service_registry: self.add_component(self.service_registry) if self.event_system: self.add_component(self.event_system) self.event_system.install(self) self.monitor = self.install(MonitorPusher, aggregator=self.metrics_aggregator, endpoint=self.monitor_endpoint, interval=5) self.add_component(rpc) rpc.request_handler = self.handle_request self.install_interface(DefaultInterface, name='lymph')
def add_service(self, **kwargs): port = self.next_port self.next_port += 1 registry = self.discovery_hub.create_registry() container = MockServiceContainer(registry=registry, events=self.events, rpc=MockRPCServer(ip='127.0.0.1', port=port, mock_network=self), metrics=Aggregator(), **kwargs) self.service_containers[container.endpoint] = container container._mock_network = self return container
def create_container(self, interface_cls=None, interface_name=None, events=None, registry=None, **kwargs): if not events: events = self.create_event_system(**kwargs) if not registry: registry = self.create_registry(**kwargs) container = ServiceContainer(events=events, registry=registry, rpc=ZmqRPCServer(), metrics=Aggregator(), **kwargs) interface = None if interface_cls: interface = container.install_interface(interface_cls, name=interface_name) container.start() self._containers.append(container) return container, interface
def test_aggregator_one_component(self): aggr = Aggregator(_get_metrics_one, name='test') self.assertIn(('dummy', 'one', { 'name': 'test' }), list(aggr.get_metrics()))
class ServiceContainer(Componentized): def __init__(self, rpc=None, registry=None, events=None, log_endpoint=None, service_name=None, debug=False, monitor_endpoint=None, pool=None): if pool is None: pool = trace.Group() super(ServiceContainer, self).__init__(error_hook=Hook('error_hook'), pool=pool) self.log_endpoint = log_endpoint self.backdoor_endpoint = None self.service_name = service_name self.fqdn = socket.getfqdn() self.http_request_hook = Hook('http_request_hook') self.server = rpc self.service_registry = registry self.events = events self.installed_interfaces = {} self.installed_plugins = [] self.debug = debug self.monitor_endpoint = monitor_endpoint self.metrics_aggregator = Aggregator(self._get_metrics, service=self.service_name, host=self.fqdn) if self.service_registry: self.add_component(self.service_registry) if self.events: self.add_component(self.events) self.events.install(self) self.monitor = self.install(MonitorPusher, aggregator=self.metrics_aggregator, endpoint=self.monitor_endpoint, interval=5) self.add_component(rpc) rpc.request_handler = self.handle_request self.install_interface(DefaultInterface, name='lymph') @classmethod def from_config(cls, config, **explicit_kwargs): kwargs = dict(config) kwargs.pop('class', None) kwargs.setdefault('monitor_endpoint', os.environ.get('LYMPH_MONITOR')) kwargs.setdefault('service_name', os.environ.get('LYMPH_SERVICE_NAME')) kwargs['registry'] = config.create_instance('registry') kwargs['rpc'] = config.create_instance('rpc', default_class=ZmqRPCServer, ip=kwargs.pop('ip', None), port=kwargs.pop('port', None)) kwargs['pool'] = config.create_instance( 'pool', default_class='lymph.core.trace:Group') for key, value in six.iteritems(explicit_kwargs): if value is not None: kwargs[key] = value return cls(**kwargs) def excepthook(self, type, value, traceback): logger.log(logging.CRITICAL, 'Uncaught exception', exc_info=(type, value, traceback)) self.error_hook((type, value, traceback)) @property def endpoint(self): return self.server.endpoint @property def identity(self): return self.server.identity def install_interface(self, cls, **kwargs): interface = cls(self, **kwargs) self.add_component(interface) self.installed_interfaces[interface.name] = interface for plugin in self.installed_plugins: plugin.on_interface_installation(interface) return interface def install_plugin(self, cls, **kwargs): plugin = self.install(cls, **kwargs) self.installed_plugins.append(plugin) return plugin def get_shared_socket_fd(self, port): try: fds = os.environ['LYMPH_SHARED_SOCKET_FDS'] except KeyError: raise NoSharedSockets() fds = json.loads(fds) try: return fds[str(port)] except KeyError: raise SocketNotCreated() @property def service_types(self): return self.installed_interfaces.keys() def subscribe(self, handler, **kwargs): return self.events.subscribe(handler, **kwargs) def unsubscribe(self, handler): self.events.unsubscribe(handler) def get_instance_description(self, interface): description = interface.get_description() description.update({ 'endpoint': self.endpoint, 'identity': self.identity, 'log_endpoint': self.log_endpoint, 'backdoor_endpoint': self.backdoor_endpoint, 'fqdn': self.fqdn, 'ip': self.server.ip, }) return description def start(self, register=True): logger.info('starting %s (%s) at %s (pid=%s)', self.service_name, ', '.join(self.service_types), self.endpoint, os.getpid()) self.on_start() self.metrics_aggregator.add_tags(identity=self.identity) if register: for interface_name, interface in six.iteritems( self.installed_interfaces): if not interface.register_with_coordinator: continue instance = ServiceInstance( **self.get_instance_description(interface)) try: self.service_registry.register(interface_name, instance) except RegistrationFailure: logger.error("registration failed %s, %s", interface_name, interface) self.stop() def stop(self, **kwargs): self.on_stop() self.pool.kill() def join(self): self.pool.join() def lookup(self, address): if '://' not in address: return self.service_registry.get(address) instance = ServiceInstance(address) return Service(address, instances=[instance]) def discover(self): return self.service_registry.discover() def emit_event(self, event_type, payload, headers=None, **kwargs): headers = headers or {} headers.setdefault('trace_id', trace.get_id()) event = Event(event_type, payload, source=self.identity, headers=headers) self.events.emit(event, **kwargs) def send_request(self, address, subject, body, headers=None): service = self.lookup(address) return self.server.send_request(service, subject, body, headers=headers) def handle_request(self, channel): interface_name, func_name = channel.request.subject.rsplit('.', 1) try: interface = self.installed_interfaces[interface_name] except KeyError: logger.warning('unsupported service type: %s', interface_name) channel.nack(True) return try: interface.handle_request(func_name, channel) except Exception: logger.exception('Request error:') exc_info = sys.exc_info() try: self.error_hook(exc_info, extra={ 'service': self.service_name, 'interface': interface_name, 'func_name': func_name, 'trace_id': trace.get_id(), }) finally: del exc_info try: channel.nack(True) except: logger.exception('failed to send automatic NACK') def _get_metrics(self): for metric in super(ServiceContainer, self)._get_metrics(): yield metric yield metrics.RawMetric('greenlets.count', len(self.pool))
class ServiceContainer(Componentized): def __init__(self, rpc=None, registry=None, events=None, log_endpoint=None, service_name=None, debug=False, monitor_endpoint=None, pool=None): if pool is None: pool = trace.Group() super(ServiceContainer, self).__init__(error_hook=Hook('error_hook'), pool=pool) self.log_endpoint = log_endpoint self.backdoor_endpoint = None self.service_name = service_name self.fqdn = socket.getfqdn() self.http_request_hook = Hook('http_request_hook') self.server = rpc self.service_registry = registry self.events = events self.installed_interfaces = {} self.installed_plugins = [] self.debug = debug self.monitor_endpoint = monitor_endpoint self.metrics_aggregator = Aggregator(self._get_metrics, service=self.service_name, host=self.fqdn) if self.service_registry: self.add_component(self.service_registry) if self.events: self.add_component(self.events) self.events.install(self) self.monitor = self.install(MonitorPusher, aggregator=self.metrics_aggregator, endpoint=self.monitor_endpoint, interval=5) self.add_component(rpc) rpc.request_handler = self.handle_request self.install_interface(DefaultInterface, name='lymph') @classmethod def from_config(cls, config, **explicit_kwargs): kwargs = dict(config) kwargs.pop('class', None) kwargs.setdefault('monitor_endpoint', os.environ.get('LYMPH_MONITOR')) kwargs.setdefault('service_name', os.environ.get('LYMPH_SERVICE_NAME')) kwargs['registry'] = config.create_instance('registry') kwargs['rpc'] = config.create_instance('rpc', default_class=ZmqRPCServer, ip=kwargs.pop('ip', None), port=kwargs.pop('port', None)) kwargs['pool'] = config.create_instance('pool', default_class='lymph.core.trace:Group') for key, value in six.iteritems(explicit_kwargs): if value is not None: kwargs[key] = value return cls(**kwargs) def excepthook(self, type, value, traceback): logger.log(logging.CRITICAL, 'Uncaught exception', exc_info=(type, value, traceback)) self.error_hook((type, value, traceback)) @property def endpoint(self): return self.server.endpoint @property def identity(self): return self.server.identity def install_interface(self, cls, **kwargs): interface = cls(self, **kwargs) self.add_component(interface) self.installed_interfaces[interface.name] = interface for plugin in self.installed_plugins: plugin.on_interface_installation(interface) return interface def install_plugin(self, cls, **kwargs): plugin = self.install(cls, **kwargs) self.installed_plugins.append(plugin) return plugin def get_shared_socket_fd(self, port): try: fds = os.environ['LYMPH_SHARED_SOCKET_FDS'] except KeyError: raise NoSharedSockets() fds = json.loads(fds) try: return fds[str(port)] except KeyError: raise SocketNotCreated() @property def service_types(self): return self.installed_interfaces.keys() def subscribe(self, handler, **kwargs): return self.events.subscribe(handler, **kwargs) def unsubscribe(self, handler): self.events.unsubscribe(handler) def get_instance_description(self, interface): description = interface.get_description() description.update({ 'endpoint': self.endpoint, 'identity': self.identity, 'log_endpoint': self.log_endpoint, 'backdoor_endpoint': self.backdoor_endpoint, 'fqdn': self.fqdn, }) return description def start(self, register=True): logger.info('starting %s (%s) at %s (pid=%s)', self.service_name, ', '.join(self.service_types), self.endpoint, os.getpid()) self.on_start() self.metrics_aggregator.add_tags(identity=self.identity) if register: for interface_name, interface in six.iteritems(self.installed_interfaces): if not interface.register_with_coordinator: continue instance = ServiceInstance(**self.get_instance_description(interface)) try: self.service_registry.register(interface_name, instance) except RegistrationFailure: logger.error("registration failed %s, %s", interface_name, interface) self.stop() def stop(self, **kwargs): self.on_stop() self.pool.kill() def join(self): self.pool.join() def lookup(self, address): if '://' not in address: return self.service_registry.get(address) instance = ServiceInstance(address) return Service(address, instances=[instance]) def discover(self): return self.service_registry.discover() def emit_event(self, event_type, payload, headers=None, **kwargs): headers = headers or {} headers.setdefault('trace_id', trace.get_id()) event = Event(event_type, payload, source=self.identity, headers=headers) self.events.emit(event, **kwargs) def send_request(self, address, subject, body, headers=None): service = self.lookup(address) return self.server.send_request(service, subject, body, headers=headers) def handle_request(self, channel): interface_name, func_name = channel.request.subject.rsplit('.', 1) try: interface = self.installed_interfaces[interface_name] except KeyError: logger.warning('unsupported service type: %s', interface_name) channel.nack(True) return try: interface.handle_request(func_name, channel) except Exception: logger.exception('Request error:') exc_info = sys.exc_info() try: self.error_hook(exc_info, extra={ 'service': self.service_name, 'interface': interface_name, 'func_name': func_name, 'trace_id': trace.get_id(), }) finally: del exc_info try: channel.nack(True) except: logger.exception('failed to send automatic NACK') def _get_metrics(self): for metric in super(ServiceContainer, self)._get_metrics(): yield metric yield metrics.RawMetric('greenlets.count', len(self.pool))
def test_aggregator_one_component(self): aggr = Aggregator(_get_metrics_one, name='test') self.assertEqual(list(aggr.get_metrics()), [('dummy', 'one', {'name': 'test'})])
class ServiceContainer(Componentized): server_cls = ZmqRPCServer def __init__(self, ip='127.0.0.1', port=None, registry=None, events=None, node_endpoint=None, log_endpoint=None, service_name=None, debug=False, monitor_endpoint=None, pool_size=None): super(ServiceContainer, self).__init__() self.node_endpoint = node_endpoint self.log_endpoint = log_endpoint self.backdoor_endpoint = None self.service_name = service_name self.fqdn = socket.getfqdn() self.service_registry = registry self.event_system = events self.error_hook = Hook() self.pool = trace.Group(size=pool_size) self.installed_interfaces = {} self.installed_plugins = [] self.debug = debug self.monitor_endpoint = monitor_endpoint self.metrics_aggregator = Aggregator(self._get_metrics, service=self.service_name, host=self.fqdn) if self.service_registry: self.add_component(self.service_registry) self.service_registry.install(self) if self.event_system: self.add_component(self.event_system) self.event_system.install(self) self.monitor = self.install(MonitorPusher, aggregator=self.metrics_aggregator, endpoint=self.monitor_endpoint, interval=5) self.server = self.install(self.server_cls, ip=ip, port=port) self.install_interface(DefaultInterface, name='lymph') @classmethod def from_config(cls, config, **explicit_kwargs): kwargs = dict(config) kwargs.pop('class', None) kwargs.setdefault('node_endpoint', os.environ.get('LYMPH_NODE')) kwargs.setdefault('monitor_endpoint', os.environ.get('LYMPH_MONITOR')) kwargs.setdefault('service_name', os.environ.get('LYMPH_SERVICE_NAME')) for key, value in six.iteritems(explicit_kwargs): if value is not None: kwargs[key] = value return cls(**kwargs) def excepthook(self, type, value, traceback): logger.log(logging.CRITICAL, 'Uncaught exception', exc_info=(type, value, traceback)) self.error_hook((type, value, traceback)) @property def endpoint(self): return self.server.endpoint @property def identity(self): return self.server.identity def spawn(self, func, *args, **kwargs): def _inner(): try: return func(*args, **kwargs) except gevent.GreenletExit: raise except: self.error_hook(sys.exc_info()) raise return self.pool.spawn(_inner) def install_interface(self, cls, **kwargs): interface = self.install(cls, **kwargs) self.installed_interfaces[interface.name] = interface for plugin in self.installed_plugins: plugin.on_interface_installation(interface) return interface def install_plugin(self, cls, **kwargs): plugin = self.install(cls, **kwargs) self.installed_plugins.append(plugin) return plugin def get_shared_socket_fd(self, port): fds = json.loads(os.environ.get('LYMPH_SHARED_SOCKET_FDS', '{}')) try: return fds[str(port)] except KeyError: raise SocketNotCreated @property def service_types(self): return self.installed_interfaces.keys() def subscribe(self, handler, **kwargs): return self.event_system.subscribe(handler, **kwargs) def unsubscribe(self, handler): self.event_system.unsubscribe(handler) def get_instance_description(self, service_type=None): return { 'endpoint': self.endpoint, 'identity': self.identity, 'log_endpoint': self.log_endpoint, 'backdoor_endpoint': self.backdoor_endpoint, 'fqdn': self.fqdn, } def start(self, register=True): logger.info('starting %s (%s) at %s (pid=%s)', self.service_name, ', '.join(self.service_types), self.endpoint, os.getpid()) self.on_start() self.metrics_aggregator.add_tags(identity=self.identity) if register: for interface_name, service in six.iteritems(self.installed_interfaces): if not service.register_with_coordinator: continue try: self.service_registry.register(interface_name) except RegistrationFailure: logger.error("registration failed %s, %s", interface_name, service) self.stop() def stop(self, **kwargs): self.on_stop() self.pool.kill() def join(self): self.pool.join() def connect(self, endpoint): for service in six.itervalues(self.installed_interfaces): service.on_connect(endpoint) return self.server.connect(endpoint) def disconnect(self, endpoint, socket=False): self.server.disconnect(endpoint, socket) for service in six.itervalues(self.installed_interfaces): service.on_disconnect(endpoint) @staticmethod def prepare_headers(headers): headers = headers or {} headers.setdefault('trace_id', trace.get_id()) return headers def lookup(self, address): if '://' not in address: return self.service_registry.get(address) instance = ServiceInstance(self, address) return Service(self, address, instances=[instance]) def discover(self): return self.service_registry.discover() def emit_event(self, event_type, payload, headers=None, **kwargs): headers = self.prepare_headers(headers) event = Event(event_type, payload, source=self.identity, headers=headers) self.event_system.emit(event, **kwargs) def send_request(self, address, subject, body, headers=None): return self.server.send_request(address, subject, body, headers=None) def _get_metrics(self): for metric in super(ServiceContainer, self)._get_metrics(): yield metric yield metrics.RawMetric('greenlets.count', len(self.pool))