def makeService(self, options):
"""
Construct a TriblerChain Crawler service.
"""
crawler_service = MultiService()
crawler_service.setName("Market")
manhole_namespace = {}
if options["manhole"] > 0:
port = options["manhole"]
manhole = manhole_tap.makeService({
'namespace':
manhole_namespace,
'telnetPort':
'tcp:%d:interface=127.0.0.1' % port,
'sshPort':
None,
'passwd':
os.path.join(os.path.dirname(__file__), 'passwd'),
})
crawler_service.addService(manhole)
reactor.callWhenRunning(self.start_tribler, options)
return crawler_service
def makeService(self, options):
tracker_service = MultiService()
tracker_service.setName("Trustchain Crawler")
def run():
crypto = ECCrypto()
dispersy = Dispersy(StandaloneEndpoint(options["port"], options["ip"]),
options["statedir"],
u'dispersy.db',
crypto)
if not dispersy.start():
raise RuntimeError("Unable to start Dispersy")
master_member = TriblerChainCommunityCrawler.get_master_members(dispersy)[0]
my_member = dispersy.get_member(private_key=crypto.key_to_bin(crypto.generate_key(u"curve25519")))
TriblerChainCommunityCrawler.init_community(dispersy, master_member, my_member)
self._stopping = False
def signal_handler(sig, frame):
msg("Received signal '%s' in %s (shutting down)" % (sig, frame))
if not self._stopping:
self._stopping = True
dispersy.stop().addCallback(lambda _: reactor.stop())
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
reactor.exitCode = 0
reactor.callWhenRunning(run)
return tracker_service
def makeService(self, options):
"""
Construct a dispersy service.
"""
tracker_service = MultiService()
tracker_service.setName("Bartercast Crawler")
# crypto
if options["crypto"] == 'NoCrypto':
crypto = NoCrypto()
else:
crypto = NoVerifyCrypto()
container = [None]
manhole_namespace = {}
if options["manhole"]:
port = options["manhole"]
manhole = manhole_tap.makeService({
'namespace':
manhole_namespace,
'telnetPort':
'tcp:%d:interface=127.0.0.1' % port,
'sshPort':
None,
'passwd':
os.path.join(os.path.dirname(__file__), 'passwd'),
})
tracker_service.addService(manhole)
manhole.startService()
def run():
# setup
dispersy = BartercastCrawler(
StandaloneEndpoint(options["port"], options["ip"]),
unicode(options["statedir"]), bool(options["silent"]), crypto)
container[0] = dispersy
manhole_namespace['dispersy'] = dispersy
self._stopping = False
def signal_handler(sig, frame):
msg("Received signal '%s' in %s (shutting down)" %
(sig, frame))
if not self._stopping:
self._stopping = True
dispersy.stop().addCallback(lambda _: reactor.stop)
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
# start
print "starting dispersy"
if not dispersy.start():
raise RuntimeError("Unable to start Dispersy")
# wait forever
reactor.exitCode = 0
reactor.callWhenRunning(run)
# TODO: exit code
return tracker_service
def makeService(self, options):
"""
Construct a tunnel helper service.
"""
tunnel_helper_service = MultiService()
tunnel_helper_service.setName("Tunnel_helper")
manhole_namespace = {}
if options["manhole"]:
port = options["manhole"]
manhole = manhole_tap.makeService({
'namespace':
manhole_namespace,
'telnetPort':
'tcp:%d:interface=127.0.0.1' % port,
'sshPort':
None,
'passwd':
os.path.join(os.path.dirname(__file__), 'passwd'),
})
tunnel_helper_service.addService(manhole)
reactor.callWhenRunning(self.start_tunnel, options)
return tunnel_helper_service
def makeService(self, options):
"""
Construct a IPv8 service.
"""
ipv8_service = MultiService()
ipv8_service.setName("AnyDex")
reactor.callWhenRunning(self.start_anydex, options)
return ipv8_service
def makeService(self, options):
"""
Construct a IPv8 service.
"""
crawler_service = MultiService()
crawler_service.setName("TrustChainCrawler")
reactor.callWhenRunning(self.start_crawler, options)
return crawler_service
def makeService(self, options):
"""
Construct a IPv8 service.
"""
crawler_service = MultiService()
crawler_service.setName("DAppCrowd")
reactor.callWhenRunning(self.start, options)
return crawler_service
def makeService(self, options):
"""
Construct a Tribler service.
"""
tribler_service = MultiService()
tribler_service.setName("Market")
reactor.callWhenRunning(self.start_tribler, options)
return tribler_service
def makeService(self, options):
"""
Construct a tracker service.
"""
tracker_service = MultiService()
tracker_service.setName("IPv8Tracker")
reactor.callWhenRunning(self.start_tracker, options)
return tracker_service
def makeService(self, options):
"""
Construct a IPv8 service.
"""
ipv8_service = MultiService()
ipv8_service.setName("IPv8Exitnode")
reactor.callWhenRunning(self.start_ipv8, options)
return ipv8_service
def makeService(self, options):
"""
Construct an OpenWallet service.
"""
ow_service = MultiService()
ow_service.setName("OpenWallet")
reactor.callWhenRunning(self.start_openwallet, options)
return ow_service
def makeService(self, options):
"""
Construct a IPv8 service.
"""
module_service = MultiService()
module_service.setName("module")
reactor.callWhenRunning(self.start, options, module_service)
return module_service
def makeService(self, options):
"""
Construct a IPv8 service.
"""
web_service = MultiService()
web_service.setName("module-loader-web")
reactor.callWhenRunning(self.start, options, web_service)
return web_service
def makeService(self, options):
"""
Construct a dispersy service.
"""
tracker_service = MultiService()
tracker_service.setName("Bartercast Crawler")
# crypto
if options["crypto"] == 'NoCrypto':
crypto = NoCrypto()
else:
crypto = NoVerifyCrypto()
container = [None]
manhole_namespace = {}
if options["manhole"]:
port = options["manhole"]
manhole = manhole_tap.makeService({
'namespace': manhole_namespace,
'telnetPort': 'tcp:%d:interface=127.0.0.1' % port,
'sshPort': None,
'passwd': os.path.join(os.path.dirname(__file__), 'passwd'),
})
tracker_service.addService(manhole)
manhole.startService()
def run():
# setup
dispersy = BartercastCrawler(StandaloneEndpoint(options["port"],
options["ip"]),
unicode(options["statedir"]),
bool(options["silent"]),
crypto)
container[0] = dispersy
manhole_namespace['dispersy'] = dispersy
self._stopping = False
def signal_handler(sig, frame):
msg("Received signal '%s' in %s (shutting down)" % (sig, frame))
if not self._stopping:
self._stopping = True
try:
dispersy.stop()
except Exception, e:
msg("Got exception when stopping dispersy: %s" % e)
reactor.stop()
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
# start
print "starting dispersy"
if not dispersy.start():
raise RuntimeError("Unable to start Dispersy")
def _makeTFTPService(self, tftp_config):
"""Create the dynamic TFTP service."""
backend = TFTPBackend(tftp_config["root"], tftp_config["generator"])
# Create a UDP server individually for each discovered network
# interface, so that we can detect the interface via which we have
# received a datagram.
tftp_services = MultiService()
tftp_services.setName("tftp")
for address in get_all_interface_addresses():
tftp_service = internet.UDPServer(
tftp_config["port"], TFTP(backend), interface=address)
tftp_service.setName(address)
tftp_service.setServiceParent(tftp_services)
return tftp_services
def _makeTFTPService(self, tftp_config):
"""Create the dynamic TFTP service."""
backend = TFTPBackend(tftp_config["root"], tftp_config["generator"])
# Create a UDP server individually for each discovered network
# interface, so that we can detect the interface via which we have
# received a datagram.
tftp_services = MultiService()
tftp_services.setName("tftp")
for address in get_all_interface_addresses():
tftp_service = internet.UDPServer(tftp_config["port"],
TFTP(backend),
interface=address)
tftp_service.setName(address)
tftp_service.setServiceParent(tftp_services)
return tftp_services
def makeService(self, options,
broadcast_protocol=None,
discovery_protocol=None):
""" Accepts options as a regular twistd plugin does. Also accepts
keyword arguments 'broadcast_protocol' for a procotcol *instance*
and 'discovery_protocol' for a protocol *instance*. Returns a
twisted.application.service.IService implementor.
"""
service_name = options.get('service-name')
resolve = options.get('resolve-domains')
port = options.get('port')
registry = options.get('registration')
service_name = options.get('service-name')
s = MultiService()
s.setName('txbonjour-%s' % (service_name,))
logging_proto = LoggingProtocol()
if broadcast_protocol is None:
broadcast_protocol = logging_proto
if discovery_protocol is None:
discovery_protocol = logging_proto
discover_service = discovery.listenBonjour(discovery_protocol,
registry,
resolve_ips=resolve,
)
discover_service.setName('discovery')
discover_service.setServiceParent(s)
def broadcast():
broadcast_service = discovery.connectBonjour(broadcast_protocol,
registry,
port,
service_name,
)
broadcast_service.setName('broadcast')
broadcast_service.setServiceParent(s)
reactor.callWhenRunning(broadcast)
return s
def makeService(self, options):
# setup logging if there is a logger.conf in the state dir or working dir
if os.path.exists(os.path.join(options["statedir"], "logger.conf")):
logging.config.fileConfig(
os.path.join(options["statedir"], "logger.conf"))
elif os.path.exists("logger.conf"):
logging.config.fileConfig("logger.conf")
else:
logging.basicConfig(
format="%(asctime)-15s [%(levelname)s] %(message)s",
level=logging.DEBUG)
tracker_service = MultiService()
tracker_service.setName("Trustchain Crawler")
def run():
crypto = ECCrypto()
dispersy = Dispersy(
StandaloneEndpoint(options["port"], options["ip"]),
options["statedir"], u'dispersy.db', crypto)
if not dispersy.start():
raise RuntimeError("Unable to start Dispersy")
master_member = TriblerChainCommunityCrawler.get_master_members(
dispersy)[0]
my_member = dispersy.get_member(private_key=crypto.key_to_bin(
crypto.generate_key(u"curve25519")))
TriblerChainCommunityCrawler.init_community(
dispersy, master_member, my_member)
self._stopping = False
def signal_handler(sig, frame):
msg("Received signal '%s' in %s (shutting down)" %
(sig, frame))
if not self._stopping:
self._stopping = True
dispersy.stop().addCallback(lambda _: reactor.stop())
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
reactor.exitCode = 0
reactor.callWhenRunning(run)
return tracker_service
def createService(options):
from tryfer.tracers import (
DebugTracer,
EndAnnotationTracer,
ZipkinTracer)
from twisted.internet import reactor
from twisted.internet.endpoints import TCP4ClientEndpoint
from scrivener import ScribeClient
from athwart.processor import HAProxyProcessor, SpanProcessor
root_service = MultiService()
root_service.setName("athwart")
tracers = []
if options["dump-mode"]:
tracers.append(EndAnnotationTracer(DebugTracer(sys.stdout)))
client = ScribeClient(TCP4ClientEndpoint(
reactor, options["scribe-host"], options["scribe-port"]))
tracers.append(ZipkinTracer(client))
haproxy_processor = HAProxyProcessor(tracers)
logstash_input = AthwartServerProtocol(
haproxy_processor,
monitor_message=options["monitor-message"],
monitor_response=options["monitor-response"])
logstash_listener = UDPServer(options["logstash-listen-port"],
logstash_input)
logstash_listener.setServiceParent(root_service)
span_processor = SpanProcessor(client)
span_input = AthwartServerProtocol(
span_processor,
monitor_message=options["monitor-message"],
monitor_response=options["monitor-response"])
span_listener = UDPServer(options["span-listen-port"], span_input)
span_listener.setServiceParent(root_service)
return root_service
def makeService(self, options):
"""
Construct a Tribler service.
"""
tribler_service = MultiService()
tribler_service.setName("Tribler")
manhole_namespace = {}
if options["manhole"] > 0:
port = options["manhole"]
manhole = manhole_tap.makeService({
'namespace': manhole_namespace,
'telnetPort': 'tcp:%d:interface=127.0.0.1' % port,
'sshPort': None,
'passwd': os.path.join(os.path.dirname(__file__), 'passwd'),
})
tribler_service.addService(manhole)
reactor.callWhenRunning(self.start_tribler, options)
return tribler_service
def add(self, name, services):
log.msg(
'bit.core.services: Services.add %s, %s' % (name, services))
if not isinstance(services, dict):
services.setName(name)
services.setServiceParent(self.collect)
self._services.append(name)
return
add = True
if name in self._multi:
plug_services = self.collect.getServiceNamed(name)
add = False
else:
plug_services = MultiService()
plug_services.setName(name)
self._multi.append(name)
for sid, s in services.items():
s.setName(sid)
s.setServiceParent(plug_services)
if add:
plug_services.setServiceParent(self.collect)
def createService(options):
"""Create a txStatsD service."""
from carbon.routers import ConsistentHashingRouter
from carbon.client import CarbonClientManager
from carbon.conf import settings
settings.MAX_QUEUE_SIZE = options["max-queue-size"]
settings.MAX_DATAPOINTS_PER_MESSAGE = options["max-datapoints-per-message"]
root_service = MultiService()
root_service.setName("statsd")
prefix = options["prefix"]
if prefix is None:
prefix = "statsd"
instance_name = options["instance-name"]
if not instance_name:
instance_name = platform.node()
# initialize plugins
plugin_metrics = []
for plugin in getPlugins(IMetricFactory):
plugin.configure(options)
plugin_metrics.append(plugin)
processor = None
if options["dump-mode"]:
# LoggingMessageProcessor supersedes
# any other processor class in "dump-mode"
assert not hasattr(log, 'info')
log.info = log.msg # for compatibility with LMP logger interface
processor = functools.partial(LoggingMessageProcessor, logger=log)
if options["statsd-compliance"]:
processor = (processor or MessageProcessor)(plugins=plugin_metrics)
input_router = Router(processor, options['routing'], root_service)
connection = InternalClient(input_router)
metrics = Metrics(connection)
else:
processor = (processor or ConfigurableMessageProcessor)(
message_prefix=prefix,
internal_metrics_prefix=prefix + "." + instance_name + ".",
plugins=plugin_metrics)
input_router = Router(processor, options['routing'], root_service)
connection = InternalClient(input_router)
metrics = ExtendedMetrics(connection)
if not options["carbon-cache-host"]:
options["carbon-cache-host"].append("127.0.0.1")
if not options["carbon-cache-port"]:
options["carbon-cache-port"].append(2004)
if not options["carbon-cache-name"]:
options["carbon-cache-name"].append(None)
reporting = ReportingService(instance_name)
reporting.setServiceParent(root_service)
reporting.schedule(report_client_manager_stats,
options["flush-interval"] / 1000,
metrics.gauge)
if options["report"] is not None:
from txstatsd import process
from twisted.internet import reactor
reporting.schedule(
process.report_reactor_stats(reactor), 60, metrics.gauge)
reports = [name.strip() for name in options["report"].split(",")]
for report_name in reports:
if report_name == "reactor":
inspector = ReactorInspectorService(reactor, metrics,
loop_time=0.05)
inspector.setServiceParent(root_service)
for reporter in getattr(process, "%s_STATS" %
report_name.upper(), ()):
reporting.schedule(reporter, 60, metrics.gauge)
# XXX Make this configurable.
router = ConsistentHashingRouter()
carbon_client = CarbonClientManager(router)
carbon_client.setServiceParent(root_service)
for host, port, name in zip(options["carbon-cache-host"],
options["carbon-cache-port"],
options["carbon-cache-name"]):
carbon_client.startClient((host, port, name))
statsd_service = StatsDService(carbon_client, input_router,
options["flush-interval"])
statsd_service.setServiceParent(root_service)
statsd_server_protocol = StatsDServerProtocol(
input_router,
monitor_message=options["monitor-message"],
monitor_response=options["monitor-response"])
listener = UDPServer(options["listen-port"], statsd_server_protocol)
listener.setServiceParent(root_service)
if options["listen-tcp-port"] is not None:
statsd_tcp_server_factory = StatsDTCPServerFactory(
input_router,
monitor_message=options["monitor-message"],
monitor_response=options["monitor-response"])
listener = TCPServer(options["listen-tcp-port"],
statsd_tcp_server_factory)
listener.setServiceParent(root_service)
httpinfo_service = httpinfo.makeService(options, processor, statsd_service)
httpinfo_service.setServiceParent(root_service)
return root_service
def createBaseService(config):
from carbon.conf import settings
from carbon.protocols import (MetricLineReceiver, MetricPickleReceiver,
MetricDatagramReceiver)
root_service = MultiService()
root_service.setName(settings.program)
use_amqp = settings.get("ENABLE_AMQP", False)
if use_amqp:
from carbon import amqp_listener
amqp_host = settings.get("AMQP_HOST", "localhost")
amqp_port = settings.get("AMQP_PORT", 5672)
amqp_user = settings.get("AMQP_USER", "guest")
amqp_password = settings.get("AMQP_PASSWORD", "guest")
amqp_verbose = settings.get("AMQP_VERBOSE", False)
amqp_vhost = settings.get("AMQP_VHOST", "/")
amqp_spec = settings.get("AMQP_SPEC", None)
amqp_exchange_name = settings.get("AMQP_EXCHANGE", "graphite")
for interface, port, protocol in ((settings.LINE_RECEIVER_INTERFACE,
settings.LINE_RECEIVER_PORT,
MetricLineReceiver),
(settings.PICKLE_RECEIVER_INTERFACE,
settings.PICKLE_RECEIVER_PORT,
MetricPickleReceiver)):
if port:
factory = ServerFactory()
factory.protocol = protocol
service = TCPServer(int(port), factory, interface=interface)
service.setServiceParent(root_service)
if settings.ENABLE_UDP_LISTENER:
service = UDPServer(int(settings.UDP_RECEIVER_PORT),
MetricDatagramReceiver(),
interface=settings.UDP_RECEIVER_INTERFACE)
service.setServiceParent(root_service)
if use_amqp:
factory = amqp_listener.createAMQPListener(
amqp_user, amqp_password,
vhost=amqp_vhost, spec=amqp_spec,
exchange_name=amqp_exchange_name,
verbose=amqp_verbose)
service = TCPClient(amqp_host, int(amqp_port), factory)
service.setServiceParent(root_service)
if settings.ENABLE_MANHOLE:
from carbon import manhole
factory = manhole.createManholeListener()
service = TCPServer(int(settings.MANHOLE_PORT), factory,
interface=settings.MANHOLE_INTERFACE)
service.setServiceParent(root_service)
# Instantiate an instrumentation service that will record metrics about
# this service.
from carbon.instrumentation import InstrumentationService
service = InstrumentationService()
service.setServiceParent(root_service)
return root_service
def createService(options):
"""Create a txStatsD service."""
from carbon.routers import ConsistentHashingRouter
from carbon.client import CarbonClientManager
from carbon.conf import settings
settings.MAX_QUEUE_SIZE = options["max-queue-size"]
settings.MAX_DATAPOINTS_PER_MESSAGE = options["max-datapoints-per-message"]
root_service = MultiService()
root_service.setName("statsd")
prefix = options["prefix"]
if prefix is None:
prefix = "statsd"
instance_name = options["instance-name"]
if not instance_name:
instance_name = platform.node()
# initialize plugins
plugin_metrics = []
for plugin in getPlugins(IMetricFactory):
plugin.configure(options)
plugin_metrics.append(plugin)
processor = None
if options["dump-mode"]:
# LoggingMessageProcessor supersedes
# any other processor class in "dump-mode"
assert not hasattr(log, 'info')
log.info = log.msg # for compatibility with LMP logger interface
processor = functools.partial(LoggingMessageProcessor, logger=log)
if options["statsd-compliance"]:
processor = (processor or MessageProcessor)(plugins=plugin_metrics)
input_router = Router(processor, options['routing'], root_service)
connection = InternalClient(input_router)
metrics = Metrics(connection)
else:
processor = (processor or ConfigurableMessageProcessor)(
message_prefix=prefix,
internal_metrics_prefix=prefix + "." + instance_name + ".",
plugins=plugin_metrics)
input_router = Router(processor, options['routing'], root_service)
connection = InternalClient(input_router)
metrics = ExtendedMetrics(connection)
if not options["carbon-cache-host"]:
options["carbon-cache-host"].append("127.0.0.1")
if not options["carbon-cache-port"]:
options["carbon-cache-port"].append(2004)
if not options["carbon-cache-name"]:
options["carbon-cache-name"].append(None)
reporting = ReportingService(instance_name)
reporting.setServiceParent(root_service)
reporting.schedule(report_client_manager_stats,
options["flush-interval"] / 1000, metrics.gauge)
if options["report"] is not None:
from txstatsd import process
from twisted.internet import reactor
reporting.schedule(process.report_reactor_stats(reactor), 60,
metrics.gauge)
reports = [name.strip() for name in options["report"].split(",")]
for report_name in reports:
if report_name == "reactor":
inspector = ReactorInspectorService(reactor,
metrics,
loop_time=0.05)
inspector.setServiceParent(root_service)
for reporter in getattr(process, "%s_STATS" % report_name.upper(),
()):
reporting.schedule(reporter, 60, metrics.gauge)
# XXX Make this configurable.
router = ConsistentHashingRouter()
carbon_client = CarbonClientManager(router)
carbon_client.setServiceParent(root_service)
for host, port, name in zip(options["carbon-cache-host"],
options["carbon-cache-port"],
options["carbon-cache-name"]):
carbon_client.startClient((host, port, name))
statsd_service = StatsDService(carbon_client, input_router,
options["flush-interval"])
statsd_service.setServiceParent(root_service)
statsd_server_protocol = StatsDServerProtocol(
input_router,
monitor_message=options["monitor-message"],
monitor_response=options["monitor-response"])
listener = UDPServer(options["listen-port"], statsd_server_protocol)
listener.setServiceParent(root_service)
if options["listen-tcp-port"] is not None:
statsd_tcp_server_factory = StatsDTCPServerFactory(
input_router,
monitor_message=options["monitor-message"],
monitor_response=options["monitor-response"])
listener = TCPServer(options["listen-tcp-port"],
statsd_tcp_server_factory)
listener.setServiceParent(root_service)
httpinfo_service = httpinfo.makeService(options, processor, statsd_service)
httpinfo_service.setServiceParent(root_service)
return root_service
