def makeService(config, reactor=reactor): parent = MultiService() basedir = FilePath(os.path.expanduser(config["basedir"])) basedir.makedirs(ignoreExistingDirectory=True) basedir.chmod(0o700) data = Data(basedir.child("config.json")) dns_server = DNSServerFactory(verbose=0) s1 = UDPServer(int(config["dns-port"]), dns.DNSDatagramProtocol(dns_server), interface=config["dns-interface"]) s1.setServiceParent(parent) s2 = TCPServer(int(config["dns-port"]), dns_server, interface=config["dns-interface"]) s2.setServiceParent(parent) s = Server(data, dns_server) s.update_records() certFile = basedir.child("tub.data").path #furlFile = basedir.child("server.furl").path t = Tub(certFile=certFile) t.setOption("keepaliveTimeout", 60) # ping after 60s of idle t.setOption("disconnectTimeout", 5*60) # disconnect/reconnect after 5m #t.setOption("logLocalFailures", True) #t.setOption("logRemoteFailures", True) #t.unsafeTracebacks = True fp = config["foolscap-port"] if not fp.startswith("tcp:"): raise usage.UsageError("I don't know how to handle non-tcp foolscap-port=") port = int(fp.split(":")[1]) assert port > 1 t.listenOn(fp) t.setLocation("tcp:%s:%d" % (config["hostname"], port)) c = Controller(data, s) cf = t.registerReference(c, furlFile=basedir.child("controller.furl").path) furl_prefix = cf[:cf.rfind("/")+1] c.set_furl_prefix(furl_prefix) t.registerNameLookupHandler(c.lookup) t.setServiceParent(parent) return parent
def build_target_redirect_udp(self, host, port): if self.service is None: return lambda *args: True port = int(port) d = defer.Deferred() self.ready.addCallback(lambda _: d) client = TwistedStatsDClient( host, port, connect_callback=lambda: d.callback(None)) protocol = StatsDClientProtocol(client) udp_service = UDPServer(0, protocol) udp_service.setServiceParent(self.service) def redirect_udp_target(metric_type, key, fields): message = self.rebuild_message(metric_type, key, fields) client.write(message) yield metric_type, key, fields return redirect_udp_target
def build_target_redirect_udp(self, host, port): if self.service is None: return lambda *args: True port = int(port) d = defer.Deferred() self.ready.addCallback(lambda _: d) client = TwistedStatsDClient.create( host, port, connect_callback=lambda: d.callback(None)) protocol = StatsDClientProtocol(client) udp_service = UDPServer(0, protocol) udp_service.setServiceParent(self.service) def redirect_udp_target(metric_type, key, fields): message = self.rebuild_message(metric_type, key, fields) client.write(message) yield metric_type, key, fields return redirect_udp_target
def setupReceivers(root_service, settings): from carbon.protocols import MetricLineReceiver, MetricPickleReceiver, MetricDatagramReceiver for protocol, interface, port in [ (MetricLineReceiver, settings.LINE_RECEIVER_INTERFACE, settings.LINE_RECEIVER_PORT), (MetricPickleReceiver, settings.PICKLE_RECEIVER_INTERFACE, settings.PICKLE_RECEIVER_PORT) ]: if port: factory = ServerFactory() factory.protocol = protocol service = TCPServer(port, factory, interface=interface) service.setServiceParent(root_service) if settings.ENABLE_UDP_LISTENER: service = UDPServer(int(settings.UDP_RECEIVER_PORT), MetricDatagramReceiver(), interface=settings.UDP_RECEIVER_INTERFACE) service.setServiceParent(root_service) if settings.ENABLE_AMQP: from carbon import amqp_listener amqp_host = settings.AMQP_HOST amqp_port = settings.AMQP_PORT amqp_user = settings.AMQP_USER amqp_password = settings.AMQP_PASSWORD amqp_verbose = settings.AMQP_VERBOSE amqp_vhost = settings.AMQP_VHOST amqp_spec = settings.AMQP_SPEC amqp_exchange_name = settings.AMQP_EXCHANGE factory = amqp_listener.createAMQPListener( amqp_user, amqp_password, vhost=amqp_vhost, spec=amqp_spec, exchange_name=amqp_exchange_name, verbose=amqp_verbose) service = TCPClient(amqp_host, amqp_port, factory) service.setServiceParent(root_service) if settings.ENABLE_MANHOLE: from carbon import manhole factory = manhole.createManholeListener() service = TCPServer(settings.MANHOLE_PORT, factory, interface=settings.MANHOLE_INTERFACE) service.setServiceParent(root_service)
def build(cls, root_service): plugin_up = cls.plugin_name.upper() interface = settings.get('%s_RECEIVER_INTERFACE' % plugin_up, None) port = int(settings.get('%s_RECEIVER_PORT' % plugin_up, 0)) protocol = cls if not port: return if hasattr(protocol, 'datagramReceived'): service = UDPServer(port, protocol(), interface=interface) service.setServiceParent(root_service) else: factory = CarbonReceiverFactory() factory.protocol = protocol service = TCPServer(port, factory, interface=interface) service.setServiceParent(root_service)
def __init__(self, **options): MultiService.__init__(self) sCtx = OpenSSLContextFactoryFactory.getFactory('fesl.ea.com') ## all port 80 services are currently merged into one server. If apache is also running, ## it must be set up to use rewriterules (for /u downloads) ## and name-based virtual hosting (for SOAP hosts) in order to redirect from port 80 to 8001 ## or whatever the server is set to run on self.addService( TCPServer(options['webPort'], gamespy.webServices.WebServer())) ## TODO: redalert3pc.natneg{1,2,3}.gamespy.com ## This is a pretty simple service that allows 2 endpoints to udp punch thru their NAT routers. ## Hosted on UDP port 27901. ## see gsnatneg.c by aluigi for details on implementation self.addService(UDPServer(27900, gamespy.master.HeartbeatMaster())) ## gsauth runs on a variety of ports in 99XY range self.addService(TCPServer(9955, gamespy.auth.GamespyAuthFactory())) address = ('cncra3-pc.fesl.ea.com', 18840) sFact = RedAlert3LoginFactory() #sFact = makeTLSFwdFactory('login.ra3cli', 'login.ra3srv')(*address) self.addService(SSLServer(addresses[0][1], sFact, sCtx)) address = ('peerchat.gamespy.com', 6667) sFact = gamespy.peerchat.PeerchatFactory() #sFact = gamespy.peerchat.ProxyPeerchatServerFactory(gameId, *address) self.addService(TCPServer(address[1], sFact)) from ...gamespy.cipher import getMsName address = (getMsName(gameId), 28910) sFact = QueryMasterFactory() #sFact = gamespy.master.ProxyMasterServerFactory(gameId, *address) self.addService(TCPServer(address[1], sFact)) address = ('gpcm.gamespy.com', 29900) sFact = gamespy.gpcm.ComradeFactory() #sFact = makeTCPFwdFactory('gamespy.gpcm.client', 'gamespy.gpcm.server')(*address) self.addService(TCPServer(address[1], sFact))
def createService(options): from tryfer.tracers import ( DebugTracer, EndAnnotationTracer, ZipkinTracer) from twisted.internet import reactor from twisted.internet.endpoints import TCP4ClientEndpoint from scrivener import ScribeClient from athwart.processor import HAProxyProcessor, SpanProcessor root_service = MultiService() root_service.setName("athwart") tracers = [] if options["dump-mode"]: tracers.append(EndAnnotationTracer(DebugTracer(sys.stdout))) client = ScribeClient(TCP4ClientEndpoint( reactor, options["scribe-host"], options["scribe-port"])) tracers.append(ZipkinTracer(client)) haproxy_processor = HAProxyProcessor(tracers) logstash_input = AthwartServerProtocol( haproxy_processor, monitor_message=options["monitor-message"], monitor_response=options["monitor-response"]) logstash_listener = UDPServer(options["logstash-listen-port"], logstash_input) logstash_listener.setServiceParent(root_service) span_processor = SpanProcessor(client) span_input = AthwartServerProtocol( span_processor, monitor_message=options["monitor-message"], monitor_response=options["monitor-response"]) span_listener = UDPServer(options["span-listen-port"], span_input) span_listener.setServiceParent(root_service) return root_service
def createBaseService(config): from carbon.conf import settings from carbon.protocols import (MetricLineReceiver, MetricPickleReceiver, MetricDatagramReceiver) root_service = CarbonRootService() root_service.setName(settings.program) use_amqp = settings.get("ENABLE_AMQP", False) if use_amqp: from carbon import amqp_listener amqp_host = settings.get("AMQP_HOST", "localhost") amqp_port = settings.get("AMQP_PORT", 5672) amqp_user = settings.get("AMQP_USER", "guest") amqp_password = settings.get("AMQP_PASSWORD", "guest") amqp_verbose = settings.get("AMQP_VERBOSE", False) amqp_vhost = settings.get("AMQP_VHOST", "/") amqp_spec = settings.get("AMQP_SPEC", None) amqp_exchange_name = settings.get("AMQP_EXCHANGE", "graphite") for interface, port, backlog, protocol in ( (settings.LINE_RECEIVER_INTERFACE, settings.LINE_RECEIVER_PORT, settings.LINE_RECEIVER_BACKLOG, MetricLineReceiver), (settings.PICKLE_RECEIVER_INTERFACE, settings.PICKLE_RECEIVER_PORT, settings.PICKLE_RECEIVER_BACKLOG, MetricPickleReceiver)): if port: factory = ServerFactory() factory.protocol = protocol service = TCPServer(int(port), factory, interface=interface, backlog=backlog) service.setServiceParent(root_service) if settings.ENABLE_UDP_LISTENER: service = UDPServer(int(settings.UDP_RECEIVER_PORT), MetricDatagramReceiver(), interface=settings.UDP_RECEIVER_INTERFACE) service.setServiceParent(root_service) if use_amqp: factory = amqp_listener.createAMQPListener( amqp_user, amqp_password, vhost=amqp_vhost, spec=amqp_spec, exchange_name=amqp_exchange_name, verbose=amqp_verbose) service = TCPClient(amqp_host, int(amqp_port), factory) service.setServiceParent(root_service) if settings.ENABLE_MANHOLE: from carbon import manhole factory = manhole.createManholeListener() service = TCPServer(int(settings.MANHOLE_PORT), factory, interface=settings.MANHOLE_INTERFACE) service.setServiceParent(root_service) if settings.USE_WHITELIST: from carbon.regexlist import WhiteList, BlackList WhiteList.read_from(settings["whitelist"]) BlackList.read_from(settings["blacklist"]) # Instantiate an instrumentation service that will record metrics about # this service. from carbon.instrumentation import InstrumentationService service = InstrumentationService() service.setServiceParent(root_service) return root_service
def createService(options): """Create a txStatsD service.""" from carbon.routers import ConsistentHashingRouter from carbon.client import CarbonClientManager from carbon.conf import settings settings.MAX_QUEUE_SIZE = options["max-queue-size"] settings.MAX_DATAPOINTS_PER_MESSAGE = options["max-datapoints-per-message"] root_service = MultiService() root_service.setName("statsd") prefix = options["prefix"] if prefix is None: prefix = "statsd" instance_name = options["instance-name"] if not instance_name: instance_name = platform.node() # initialize plugins plugin_metrics = [] for plugin in getPlugins(IMetricFactory): plugin.configure(options) plugin_metrics.append(plugin) processor = None if options["dump-mode"]: # LoggingMessageProcessor supersedes # any other processor class in "dump-mode" assert not hasattr(log, 'info') log.info = log.msg # for compatibility with LMP logger interface processor = functools.partial(LoggingMessageProcessor, logger=log) if options["statsd-compliance"]: processor = (processor or MessageProcessor)(plugins=plugin_metrics) input_router = Router(processor, options['routing'], root_service) connection = InternalClient(input_router) metrics = Metrics(connection) else: processor = (processor or ConfigurableMessageProcessor)( message_prefix=prefix, internal_metrics_prefix=prefix + "." + instance_name + ".", plugins=plugin_metrics) input_router = Router(processor, options['routing'], root_service) connection = InternalClient(input_router) metrics = ExtendedMetrics(connection) if not options["carbon-cache-host"]: options["carbon-cache-host"].append("127.0.0.1") if not options["carbon-cache-port"]: options["carbon-cache-port"].append(2004) if not options["carbon-cache-name"]: options["carbon-cache-name"].append(None) reporting = ReportingService(instance_name) reporting.setServiceParent(root_service) reporting.schedule(report_client_manager_stats, options["flush-interval"] / 1000, metrics.gauge) if options["report"] is not None: from txstatsd import process from twisted.internet import reactor reporting.schedule(process.report_reactor_stats(reactor), 60, metrics.gauge) reports = [name.strip() for name in options["report"].split(",")] for report_name in reports: if report_name == "reactor": inspector = ReactorInspectorService(reactor, metrics, loop_time=0.05) inspector.setServiceParent(root_service) for reporter in getattr(process, "%s_STATS" % report_name.upper(), ()): reporting.schedule(reporter, 60, metrics.gauge) # XXX Make this configurable. router = ConsistentHashingRouter() carbon_client = CarbonClientManager(router) carbon_client.setServiceParent(root_service) for host, port, name in zip(options["carbon-cache-host"], options["carbon-cache-port"], options["carbon-cache-name"]): carbon_client.startClient((host, port, name)) statsd_service = StatsDService(carbon_client, input_router, options["flush-interval"]) statsd_service.setServiceParent(root_service) statsd_server_protocol = StatsDServerProtocol( input_router, monitor_message=options["monitor-message"], monitor_response=options["monitor-response"]) listener = UDPServer(options["listen-port"], statsd_server_protocol) listener.setServiceParent(root_service) if options["listen-tcp-port"] is not None: statsd_tcp_server_factory = StatsDTCPServerFactory( input_router, monitor_message=options["monitor-message"], monitor_response=options["monitor-response"]) listener = TCPServer(options["listen-tcp-port"], statsd_tcp_server_factory) listener.setServiceParent(root_service) httpinfo_service = httpinfo.makeService(options, processor, statsd_service) httpinfo_service.setServiceParent(root_service) return root_service
def createService(options): """Create a txStatsD service.""" from carbon.routers import ConsistentHashingRouter from carbon.client import CarbonClientManager from carbon.conf import settings settings.MAX_QUEUE_SIZE = options["max-queue-size"] settings.MAX_DATAPOINTS_PER_MESSAGE = options["max-datapoints-per-message"] root_service = MultiService() root_service.setName("statsd") prefix = options["prefix"] if prefix is None: prefix = "statsd" instance_name = options["instance-name"] if not instance_name: instance_name = platform.node() # initialize plugins plugin_metrics = [] for plugin in getPlugins(IMetricFactory): plugin.configure(options) plugin_metrics.append(plugin) processor = None if options["dump-mode"]: # LoggingMessageProcessor supersedes # any other processor class in "dump-mode" assert not hasattr(log, 'info') log.info = log.msg # for compatibility with LMP logger interface processor = functools.partial(LoggingMessageProcessor, logger=log) if options["statsd-compliance"]: processor = (processor or MessageProcessor)(plugins=plugin_metrics) input_router = Router(processor, options['routing'], root_service) connection = InternalClient(input_router) metrics = Metrics(connection) else: processor = (processor or ConfigurableMessageProcessor)( message_prefix=prefix, internal_metrics_prefix=prefix + "." + instance_name + ".", plugins=plugin_metrics) input_router = Router(processor, options['routing'], root_service) connection = InternalClient(input_router) metrics = ExtendedMetrics(connection) if not options["carbon-cache-host"]: options["carbon-cache-host"].append("127.0.0.1") if not options["carbon-cache-port"]: options["carbon-cache-port"].append(2004) if not options["carbon-cache-name"]: options["carbon-cache-name"].append(None) reporting = ReportingService(instance_name) reporting.setServiceParent(root_service) reporting.schedule(report_client_manager_stats, options["flush-interval"] / 1000, metrics.gauge) if options["report"] is not None: from txstatsd import process from twisted.internet import reactor reporting.schedule( process.report_reactor_stats(reactor), 60, metrics.gauge) reports = [name.strip() for name in options["report"].split(",")] for report_name in reports: if report_name == "reactor": inspector = ReactorInspectorService(reactor, metrics, loop_time=0.05) inspector.setServiceParent(root_service) for reporter in getattr(process, "%s_STATS" % report_name.upper(), ()): reporting.schedule(reporter, 60, metrics.gauge) # XXX Make this configurable. router = ConsistentHashingRouter() carbon_client = CarbonClientManager(router) carbon_client.setServiceParent(root_service) for host, port, name in zip(options["carbon-cache-host"], options["carbon-cache-port"], options["carbon-cache-name"]): carbon_client.startClient((host, port, name)) statsd_service = StatsDService(carbon_client, input_router, options["flush-interval"]) statsd_service.setServiceParent(root_service) statsd_server_protocol = StatsDServerProtocol( input_router, monitor_message=options["monitor-message"], monitor_response=options["monitor-response"]) listener = UDPServer(options["listen-port"], statsd_server_protocol) listener.setServiceParent(root_service) if options["listen-tcp-port"] is not None: statsd_tcp_server_factory = StatsDTCPServerFactory( input_router, monitor_message=options["monitor-message"], monitor_response=options["monitor-response"]) listener = TCPServer(options["listen-tcp-port"], statsd_tcp_server_factory) listener.setServiceParent(root_service) httpinfo_service = httpinfo.makeService(options, processor, statsd_service) httpinfo_service.setServiceParent(root_service) return root_service
factory[key].update(value, *args) class SketchFactory(object): def __init__(self, clazz, *args, **kargs): self.clazz = clazz self.args = args self.kargs = kargs self.table = {} def __getitem__(self, key): if key not in self.table: log.msg("Create: '%s' [%s]" % (key, self.clazz.__name__)) self.table[key] = self.clazz(key, *self.args, **self.kargs) return self.table[key] dispatcher = Dispatcher() dispatcher.register("hh", SketchFactory(SpaceSaving)) sketch_udp_protocol = SketchUDPServerProtocol(MessageProcessor(dispatcher)) server = UDPServer(LISTEN_PORT, sketch_udp_protocol) application = Application("sketch") server.setServiceParent(application)
def __init__(self, parent, interval, host, port): p = QueryProtocol(host, port) p.dispatch = parent.events.dispatch p.interval = interval UDPServer.__init__(self, 0, p)