def createRelayService(config): from carbon.routers import RelayRulesRouter, ConsistentHashingRouter, AggregatedConsistentHashingRouter from carbon.client import CarbonClientManager from carbon.conf import settings from carbon import events root_service = createBaseService(config) # Configure application components if settings.RELAY_METHOD == 'rules': router = RelayRulesRouter(settings["relay-rules"]) elif settings.RELAY_METHOD == 'consistent-hashing': router = ConsistentHashingRouter( settings.REPLICATION_FACTOR, diverse_replicas=settings.DIVERSE_REPLICAS) elif settings.RELAY_METHOD == 'aggregated-consistent-hashing': from carbon.aggregator.rules import RuleManager RuleManager.read_from(settings["aggregation-rules"]) router = AggregatedConsistentHashingRouter(RuleManager, settings.REPLICATION_FACTOR) client_manager = CarbonClientManager(router) client_manager.setServiceParent(root_service) events.metricReceived.addHandler(client_manager.sendDatapoint) events.metricGenerated.addHandler(client_manager.sendDatapoint) if not settings.DESTINATIONS: raise CarbonConfigException( "Required setting DESTINATIONS is missing from carbon.conf") for destination in util.parseDestinations(settings.DESTINATIONS): client_manager.startClient(destination) return root_service
def createAggregatorService(config): from carbon.aggregator import receiver from carbon.aggregator.rules import RuleManager from carbon.routers import ConsistentHashingRouter from carbon.client import CarbonClientManager from carbon.rewrite import RewriteRuleManager from carbon.conf import settings from carbon import events root_service = createBaseService(config) # Configure application components router = ConsistentHashingRouter( settings.REPLICATION_FACTOR, diverse_replicas=settings.DIVERSE_REPLICAS) client_manager = CarbonClientManager(router) client_manager.setServiceParent(root_service) events.metricReceived.addHandler(receiver.process) events.metricGenerated.addHandler(client_manager.sendDatapoint) RuleManager.read_from(settings["aggregation-rules"]) if exists(settings["rewrite-rules"]): RewriteRuleManager.read_from(settings["rewrite-rules"]) if not settings.DESTINATIONS: raise CarbonConfigException( "Required setting DESTINATIONS is missing from carbon.conf") for destination in util.parseDestinations(settings.DESTINATIONS): client_manager.startClient(destination) return root_service
def createRelayService(config): from carbon.routers import RelayRulesRouter, ConsistentHashingRouter, AggregatedConsistentHashingRouter from carbon.client import CarbonClientManager from carbon.conf import settings from carbon import events root_service = createBaseService(config) # Configure application components if settings.RELAY_METHOD == 'rules': router = RelayRulesRouter(settings["relay-rules"]) elif settings.RELAY_METHOD == 'consistent-hashing': router = ConsistentHashingRouter(settings.REPLICATION_FACTOR) elif settings.RELAY_METHOD == 'aggregated-consistent-hashing': from carbon.aggregator.rules import RuleManager RuleManager.read_from(settings["aggregation-rules"]) router = AggregatedConsistentHashingRouter(RuleManager, settings.REPLICATION_FACTOR) client_manager = CarbonClientManager(router) client_manager.setServiceParent(root_service) events.metricReceived.addHandler(client_manager.sendDatapoint) events.metricGenerated.addHandler(client_manager.sendDatapoint) events.specialMetricReceived.addHandler(client_manager.sendHighPriorityDatapoint) events.specialMetricGenerated.addHandler(client_manager.sendHighPriorityDatapoint) if not settings.DESTINATIONS: raise CarbonConfigException("Required setting DESTINATIONS is missing from carbon.conf") for destination in util.parseDestinations(settings.DESTINATIONS): client_manager.startClient(destination) return root_service
def createAggregatorService(config): from carbon.aggregator import receiver from carbon.aggregator.rules import RuleManager from carbon.routers import ConsistentHashingRouter from carbon.client import CarbonClientManager from carbon.rewrite import RewriteRuleManager from carbon.conf import settings from carbon import events root_service = createBaseService(config) # Configure application components router = ConsistentHashingRouter() client_manager = CarbonClientManager(router) client_manager.setServiceParent(root_service) events.metricReceived.addHandler(receiver.process) events.metricGenerated.addHandler(client_manager.sendDatapoint) RuleManager.read_from(settings["aggregation-rules"]) if exists(settings["rewrite-rules"]): RewriteRuleManager.read_from(settings["rewrite-rules"]) if not settings.DESTINATIONS: raise CarbonConfigException("Required setting DESTINATIONS is missing from carbon.conf") for destination in util.parseDestinations(settings.DESTINATIONS): client_manager.startClient(destination) return root_service
def createService(options): """Create a txStatsD service.""" from carbon.routers import ConsistentHashingRouter from carbon.client import CarbonClientManager from carbon.conf import settings settings.MAX_QUEUE_SIZE = options["max-queue-size"] settings.MAX_DATAPOINTS_PER_MESSAGE = options["max-datapoints-per-message"] root_service = MultiService() root_service.setName("statsd") prefix = options["prefix"] if prefix is None: prefix = "statsd" instance_name = options["instance-name"] if not instance_name: instance_name = platform.node() # initialize plugins plugin_metrics = [] for plugin in getPlugins(IMetricFactory): plugin.configure(options) plugin_metrics.append(plugin) processor = None if options["dump-mode"]: # LoggingMessageProcessor supersedes # any other processor class in "dump-mode" assert not hasattr(log, 'info') log.info = log.msg # for compatibility with LMP logger interface processor = functools.partial(LoggingMessageProcessor, logger=log) if options["statsd-compliance"]: processor = (processor or MessageProcessor)(plugins=plugin_metrics) input_router = Router(processor, options['routing'], root_service) connection = InternalClient(input_router) metrics = Metrics(connection) else: processor = (processor or ConfigurableMessageProcessor)( message_prefix=prefix, internal_metrics_prefix=prefix + "." + instance_name + ".", plugins=plugin_metrics) input_router = Router(processor, options['routing'], root_service) connection = InternalClient(input_router) metrics = ExtendedMetrics(connection) if not options["carbon-cache-host"]: options["carbon-cache-host"].append("127.0.0.1") if not options["carbon-cache-port"]: options["carbon-cache-port"].append(2004) if not options["carbon-cache-name"]: options["carbon-cache-name"].append(None) reporting = ReportingService(instance_name) reporting.setServiceParent(root_service) reporting.schedule(report_client_manager_stats, options["flush-interval"] / 1000, metrics.gauge) if options["report"] is not None: from txstatsd import process from twisted.internet import reactor reporting.schedule(process.report_reactor_stats(reactor), 60, metrics.gauge) reports = [name.strip() for name in options["report"].split(",")] for report_name in reports: if report_name == "reactor": inspector = ReactorInspectorService(reactor, metrics, loop_time=0.05) inspector.setServiceParent(root_service) for reporter in getattr(process, "%s_STATS" % report_name.upper(), ()): reporting.schedule(reporter, 60, metrics.gauge) # XXX Make this configurable. router = ConsistentHashingRouter() carbon_client = CarbonClientManager(router) carbon_client.setServiceParent(root_service) for host, port, name in zip(options["carbon-cache-host"], options["carbon-cache-port"], options["carbon-cache-name"]): carbon_client.startClient((host, port, name)) statsd_service = StatsDService(carbon_client, input_router, options["flush-interval"]) statsd_service.setServiceParent(root_service) statsd_server_protocol = StatsDServerProtocol( input_router, monitor_message=options["monitor-message"], monitor_response=options["monitor-response"]) listener = UDPServer(options["listen-port"], statsd_server_protocol) listener.setServiceParent(root_service) if options["listen-tcp-port"] is not None: statsd_tcp_server_factory = StatsDTCPServerFactory( input_router, monitor_message=options["monitor-message"], monitor_response=options["monitor-response"]) listener = TCPServer(options["listen-tcp-port"], statsd_tcp_server_factory) listener.setServiceParent(root_service) httpinfo_service = httpinfo.makeService(options, processor, statsd_service) httpinfo_service.setServiceParent(root_service) return root_service
def createService(options): """Create a txStatsD service.""" from carbon.routers import ConsistentHashingRouter from carbon.client import CarbonClientManager from carbon.conf import settings settings.MAX_QUEUE_SIZE = options["max-queue-size"] settings.MAX_DATAPOINTS_PER_MESSAGE = options["max-datapoints-per-message"] root_service = MultiService() root_service.setName("statsd") prefix = options["prefix"] if prefix is None: prefix = "statsd" instance_name = options["instance-name"] if not instance_name: instance_name = platform.node() # initialize plugins plugin_metrics = [] for plugin in getPlugins(IMetricFactory): plugin.configure(options) plugin_metrics.append(plugin) processor = None if options["dump-mode"]: # LoggingMessageProcessor supersedes # any other processor class in "dump-mode" assert not hasattr(log, 'info') log.info = log.msg # for compatibility with LMP logger interface processor = functools.partial(LoggingMessageProcessor, logger=log) if options["statsd-compliance"]: processor = (processor or MessageProcessor)(plugins=plugin_metrics) input_router = Router(processor, options['routing'], root_service) connection = InternalClient(input_router) metrics = Metrics(connection) else: processor = (processor or ConfigurableMessageProcessor)( message_prefix=prefix, internal_metrics_prefix=prefix + "." + instance_name + ".", plugins=plugin_metrics) input_router = Router(processor, options['routing'], root_service) connection = InternalClient(input_router) metrics = ExtendedMetrics(connection) if not options["carbon-cache-host"]: options["carbon-cache-host"].append("127.0.0.1") if not options["carbon-cache-port"]: options["carbon-cache-port"].append(2004) if not options["carbon-cache-name"]: options["carbon-cache-name"].append(None) reporting = ReportingService(instance_name) reporting.setServiceParent(root_service) reporting.schedule(report_client_manager_stats, options["flush-interval"] / 1000, metrics.gauge) if options["report"] is not None: from txstatsd import process from twisted.internet import reactor reporting.schedule( process.report_reactor_stats(reactor), 60, metrics.gauge) reports = [name.strip() for name in options["report"].split(",")] for report_name in reports: if report_name == "reactor": inspector = ReactorInspectorService(reactor, metrics, loop_time=0.05) inspector.setServiceParent(root_service) for reporter in getattr(process, "%s_STATS" % report_name.upper(), ()): reporting.schedule(reporter, 60, metrics.gauge) # XXX Make this configurable. router = ConsistentHashingRouter() carbon_client = CarbonClientManager(router) carbon_client.setServiceParent(root_service) for host, port, name in zip(options["carbon-cache-host"], options["carbon-cache-port"], options["carbon-cache-name"]): carbon_client.startClient((host, port, name)) statsd_service = StatsDService(carbon_client, input_router, options["flush-interval"]) statsd_service.setServiceParent(root_service) statsd_server_protocol = StatsDServerProtocol( input_router, monitor_message=options["monitor-message"], monitor_response=options["monitor-response"]) listener = UDPServer(options["listen-port"], statsd_server_protocol) listener.setServiceParent(root_service) if options["listen-tcp-port"] is not None: statsd_tcp_server_factory = StatsDTCPServerFactory( input_router, monitor_message=options["monitor-message"], monitor_response=options["monitor-response"]) listener = TCPServer(options["listen-tcp-port"], statsd_tcp_server_factory) listener.setServiceParent(root_service) httpinfo_service = httpinfo.makeService(options, processor, statsd_service) httpinfo_service.setServiceParent(root_service) return root_service