def __init__(self, bucket=None, prefix=None,
access_key_id=None, secret_key=None):
self.s3con = boto.connect_s3(access_key_id, secret_key)
self.bucket = self.s3con.get_bucket(bucket)
self.prefix = prefix or ''
DiagnosticsEndpoints.__init__(self)
http.HttpServer.__init__(self)
def __init__(self, zk_handle, domain, ttl, soa_data):
self.zkclient = zk_handle
self.domain = domain.strip('.')
self.soa_data = soa_data
self.ttl = ttl
http.HttpServer.__init__(self)
DiagnosticsEndpoints.__init__(self)
metrics.MetricsEndpoints.__init__(self)
def __init__(self, zk_basepath, subdomain, base_domain):
# Mapping of zookeeper URIs to matching TwitterKazooClient
self.zk_connections = {}
self.zk_basepath = zk_basepath
self.job_re = re.compile(JOB_RE % {'subdomain': subdomain,
'domainname': base_domain})
DiagnosticsEndpoints.__init__(self)
http.HttpServer.__init__(self)
def __init__(self, zk, zk_basepath, scheduler_url, subdomain, base_domain):
self.zkclient = zk
self.zk_basepath = zk_basepath
self.scheduler_url = scheduler_url
job_re = JOB_RE % {'subdomain': subdomain,
'domainname': base_domain}
log.debug("Job hostname regex: %s", job_re)
self.job_re = re.compile(job_re)
DiagnosticsEndpoints.__init__(self)
http.HttpServer.__init__(self)
def main(_, opts):
stats = StatsServer(opts.iface, opts.zookeeper_port,
opts.aggregation_depth, opts.max_results,
opts.max_queued_requests, opts.max_queued_replies,
opts.max_queued_events)
log.info("Starting with opts: %s" % (opts))
signal.signal(signal.SIGINT, signal.SIG_DFL)
process = ProcessOptions()
if opts.niceness >= 0:
process.set_niceness(opts.niceness)
if opts.cpu_affinity:
process.set_cpu_affinity(opts.cpu_affinity)
server = Server()
server.mount_routes(DiagnosticsEndpoints())
server.mount_routes(stats)
server.run(opts.http_addr, opts.http_port)
while True:
time.sleep(10)
def configure_server(task_observer):
bottle_wrapper = BottleObserver(task_observer)
root_metrics = RootMetrics()
server = HttpServer()
server.mount_routes(bottle_wrapper)
server.mount_routes(DiagnosticsEndpoints())
server.mount_routes(VarsEndpoint())
register_build_properties(root_metrics)
register_diagnostics(root_metrics)
return server
def setup_function(self):
assert self._thread is None, "Attempting to call start() after server has been started!"
options = app.get_options()
parent = self
self.mount_routes(DiagnosticsEndpoints())
class RootServerThread(threading.Thread):
def run(self):
rs = parent
rs.run(
options.twitter_common_http_root_server_host,
options.twitter_common_http_root_server_port,
server=options.twitter_common_http_root_server_framework)
if options.twitter_common_http_root_server_enabled:
self._thread = RootServerThread()
self._thread.start()
def main(_, opts):
if opts.version:
sys.stdout.write("%s\n" % __version__)
sys.exit(0)
# set proc options before we spawn threads
process = ProcessOptions()
if opts.niceness >= 0:
process.set_niceness(opts.niceness)
if opts.cpu_affinity:
process.set_cpu_affinity(opts.cpu_affinity)
if opts.sampling < 0 or opts.sampling > 1:
sys.stdout.write("--sampling takes values within [0, 1]\n")
sys.exit(1)
stats = StatsServer(opts.iface,
opts.zookeeper_port,
opts.aggregation_depth,
opts.max_results,
opts.max_queued_requests,
opts.max_queued_replies,
opts.max_queued_events,
sampling=opts.sampling,
include_bytes=not opts.exclude_bytes)
log.info("Starting with opts: %s" % (opts))
signal.signal(signal.SIGINT, signal.SIG_DFL)
server = Server()
server.mount_routes(DiagnosticsEndpoints())
server.mount_routes(stats)
server.run(opts.http_addr, opts.http_port)
stats.sniffer.join()
def main(args, opts):
if args:
print("ERROR: unrecognized arguments: %s\n" % (" ".join(args)),
file=sys.stderr)
app.help()
sys.exit(1)
root_server = HttpServer()
root_server.mount_routes(DiagnosticsEndpoints())
task_observer = TaskObserver(opts.root)
task_observer.start()
bottle_wrapper = BottleObserver(task_observer)
root_server.mount_routes(bottle_wrapper)
def run():
root_server.run('0.0.0.0', opts.port, 'cherrypy')
et = ExceptionalThread(target=run)
et.daemon = True
et.start()
et.join()
def setup_function(self):
assert self._thread is None, "Attempting to call start() after server has been started!"
options = app.get_options()
parent = self
self.mount_routes(DiagnosticsEndpoints())
if not options.twitter_common_http_root_server_disable_lifecycle:
self.mount_routes(LifecycleEndpoints())
class RootServerThread(ExceptionalThread):
def __init__(self):
super(RootServerThread, self).__init__()
self.daemon = True
def run(self):
rs = parent
rs.run(
options.twitter_common_http_root_server_host,
options.twitter_common_http_root_server_port,
server=options.twitter_common_http_root_server_framework)
if options.twitter_common_http_root_server_enabled:
self._thread = RootServerThread()
self._thread.start()
def test_endpoints():
class Server(HttpServer):
pass
def free_port():
s = socket.socket()
s.bind(('', 0))
return s.getsockname()[1]
server_addr = "127.0.0.1"
server_port = free_port()
bottle.ServerAdapter.quiet = True
timer = FakeTimer()
stats = StatsServer("yolo", 2181, 1, 10, 100, 100, 100, False, timer)
server = Server()
server.mount_routes(DiagnosticsEndpoints())
server.mount_routes(stats)
# FIXME(rgs): how do you get a free port in Travis?
worker = threading.Thread(target=server.run, args=(server_addr, server_port))
worker.setDaemon(True)
worker.start()
consume_packets('set_data', stats.sniffer)
def ping(server, port):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect((server, port))
return True
except socket.error:
return False
finally:
sock.close()
for i in range(0, 10):
if ping(server_addr, server_port):
break
time.sleep(1)
else:
raise Exception("server didn't come up")
conn = httplib.HTTPConnection("127.0.0.1:%d" % server_port)
conn.request("GET", "/json/info")
resp = conn.getresponse()
assert resp.status == 200
assert "uptime" in resp.read()
timer.tick()
# wait for stats
while True:
stats.wakeup()
if stats.has_stats:
break
conn.request("GET", "/json/paths")
resp = conn.getresponse()
assert resp.status == 200
paths = json.loads(resp.read())
assert paths["ExistsRequest/load-testing"] == 4
assert paths["ExistsRequestBytes/load-testing"] == 112
assert paths["SetDataRequest/load-testing"] == 20
assert paths["SetDataRequestBytes/load-testing"] == 10999
assert paths["reads/"] == 12
assert paths["reads/load-testing"] == 4
assert paths["readsBytes/"] == 3046
assert paths["readsBytes/load-testing"] == 112
assert paths["total/readBytes"] == 3158
assert paths["total/reads"] == 16
assert paths["total/writeBytes"] == 10999
assert paths["total/writes"] == 20
assert paths["writes/load-testing"] == 20
assert paths["writesBytes/load-testing"] == 10999
conn.request("GET", "/json/ips")
resp = conn.getresponse()
assert resp.status == 200
ips = json.loads(resp.read())
assert ips["per_ip/total/writes"] == 20
assert ips["per_ip/ConnectRequest:127.0.0.1"] == 6
conn.request("GET", "/json/auths")
resp = conn.getresponse()
assert resp.status == 200
auths = json.loads(resp.read())
assert auths["per_auth/ConnectRequest:noauth"] == 6
conn.request("GET", "/json/auths-dump")
resp = conn.getresponse()
assert resp.status == 200
auths_dump = json.loads(resp.read())
assert auths_dump["127.0.0.1:59819"] == "noauth"
assert auths_dump["127.0.0.1:59817"] == "noauth"
conn.close()