def log_expvars(cluster_config, folder_name):
"""
usage: log_expvars.py"
"""
# Get gateload ips from ansible inventory
lgs_host_vars = hosts_for_tag(cluster_config, "load_generators")
lgs = [lg["ansible_host"] for lg in lgs_host_vars]
lgs_expvar_endpoints = [lg + ":9876/debug/vars" for lg in lgs]
log_info("Monitoring gateloads until they finish: {}".format("\n".join(lgs_expvar_endpoints)))
# Get sync_gateway ips from ansible inventory
sgs_host_vars = hosts_for_tag(cluster_config, "sync_gateways")
sgs = [sgv["ansible_host"] for sgv in sgs_host_vars]
sgs_expvar_endpoints = [sg + ":4985/_expvar" for sg in sgs]
log_info("Monitoring sync_gateways: {}".format("\n".join(sgs_expvar_endpoints)))
# Verify that sync gateway expvar endpoints are reachable
wait_for_endpoints_alive_or_raise(sgs_expvar_endpoints)
# Wait until the gateload expvar endpoints are up, or raise an exception and abort
wait_for_endpoints_alive_or_raise(lgs_expvar_endpoints)
start_time = time.time()
gateload_results = OrderedDict()
sync_gateway_results = OrderedDict()
gateload_is_running = True
while gateload_is_running:
# Caputure expvars for gateloads
for endpoint in lgs_expvar_endpoints:
try:
write_expvars(gateload_results, endpoint)
except ConnectionError as he:
# connection to gateload expvars has been closed
log_info("Gateload {} no longer reachable. Writing expvars to {}".format(endpoint, folder_name))
dump_results(folder_name, gateload_results, sync_gateway_results)
gateload_is_running = False
# Capture expvars for sync_gateways
for endpoint in sgs_expvar_endpoints:
try:
write_expvars(sync_gateway_results, endpoint)
except ConnectionError as he:
# Should not happen unless sg crashes
log_info(he)
log_info("ERROR: sync_gateway not reachable. Dumping results to {}".format(folder_name))
dump_results(folder_name, gateload_results, sync_gateway_results)
log_info("Elapsed: {} minutes".format((time.time() - start_time) / 60.0))
time.sleep(30)
def log_expvars(folder_name):
usage = """
usage: log_expvars.py"
"""
# Get gateload ips from ansible inventory
lgs_host_vars = hosts_for_tag("load_generators")
lgs = [lg["ansible_host"] for lg in lgs_host_vars]
print("Monitoring gateloads: {}".format(lgs))
lgs_expvar_endpoints = [lg + ":9876/debug/vars" for lg in lgs]
# Get sync_gateway ips from ansible inventory
sgs_host_vars = hosts_for_tag("sync_gateways")
sgs = [sgv["ansible_host"] for sgv in sgs_host_vars]
print("Monitoring sync_gateways: {}".format(sgs))
sgs_expvar_endpoints = [sg + ":4985/_expvar" for sg in sgs]
start_time = time.time()
gateload_results = OrderedDict()
sync_gateway_results = OrderedDict()
gateload_is_running = True
while gateload_is_running:
# Caputure expvars for gateloads
for endpoint in lgs_expvar_endpoints:
try:
write_expvars(gateload_results, endpoint)
except ConnectionError as he:
# connection to gateload expvars has been closed
print(he)
print("Gateload no longer reachable. Writing expvars ...")
dump_results(folder_name, gateload_results, sync_gateway_results)
gateload_is_running = False
# Capture expvars for sync_gateways
for endpoint in sgs_expvar_endpoints:
try:
write_expvars(sync_gateway_results, endpoint)
except ConnectionError as he:
# Should not happen unless sg crashes
print(he)
print("ERROR: sync_gateway not reachable. Dumping results")
dump_results(folder_name, gateload_results, sync_gateway_results)
print("Elapsed: {} minutes".format((time.time() - start_time) / 60.0))
time.sleep(30)
if __name__ == "__main__":
usage = """
usage: python monitor_gateload.py"
"""
try:
cluster_conf = os.environ["CLUSTER_CONFIG"]
except KeyError as ke:
log_info(
"Make sure CLUSTER_CONFIG is defined and pointing to the configuration you would like to provision"
)
raise KeyError(
"CLUSTER_CONFIG not defined. Unable to provision cluster.")
# Get gateload ips from ansible inventory
lgs_host_vars = hosts_for_tag(cluster_conf, "load_generators")
lgs = [lg["ansible_host"] for lg in lgs_host_vars]
if len(lgs) == 0:
print("No gateloads to monitor in 'provisioning_config'")
sys.exit(1)
print("Monitoring gateloads: {}".format(lgs))
lgs_with_port = [lg + ":9876" for lg in lgs]
lgs_joined = ",".join(lgs_with_port)
expvars = [
"gateload.ops.PushToSubscriberInteractive.p95",
"gateload.ops.PushToSubscriberInteractive.p99",
"gateload.total_doc_pulled", "gateload.total_doc_pushed",
"gateload.user_active", "gateload.user_awake",
def log_expvars(cluster_config, folder_name, sleep_time=30):
"""
usage: log_expvars.py"
"""
finished_successfully = True
try:
# Get gateload ips from ansible inventory
lgs_host_vars = hosts_for_tag(cluster_config, "load_generators")
lgs = [lg["ansible_host"] for lg in lgs_host_vars]
lgs_expvar_endpoints = [lg + ":9876/debug/vars" for lg in lgs]
log_info("Monitoring gateloads until they finish: {}".format(
"\n".join(lgs_expvar_endpoints)))
# Get sync_gateway ips from ansible inventory
sgs_host_vars = hosts_for_tag(cluster_config, "sync_gateways")
sgs = [sgv["ansible_host"] for sgv in sgs_host_vars]
sgs_expvar_endpoints = [sg + ":4985/_expvar" for sg in sgs]
log_info("Monitoring sync_gateways: {}".format(
"\n".join(sgs_expvar_endpoints)))
# Verify that sync gateway expvar endpoints are reachable
wait_for_endpoints_alive_or_raise(sgs_expvar_endpoints)
# Wait until the gateload expvar endpoints are up, or raise an exception and abort
wait_for_endpoints_alive_or_raise(lgs_expvar_endpoints)
start_time = time.time()
gateload_results = OrderedDict()
sync_gateway_results = OrderedDict()
gateload_is_running = True
sg_is_running = True
while gateload_is_running and sg_is_running:
# Capture expvars for gateloads
for endpoint in lgs_expvar_endpoints:
try:
log_info("Collecting gateload expvars {}".format(endpoint))
write_expvars(gateload_results, endpoint)
dump_results(folder_name, gateload_results,
sync_gateway_results)
except RequestException as re:
# connection to gateload expvars has been closed
log_info(re)
log_info(
"Error: {}. Gateload {} no longer reachable. Writing expvars to {}"
.format(re, endpoint, folder_name))
dump_results(folder_name, gateload_results,
sync_gateway_results)
gateload_is_running = False
# Capture expvars for sync_gateways
for endpoint in sgs_expvar_endpoints:
try:
log_info("Collecting sg expvars {}".format(endpoint))
write_expvars(sync_gateway_results, endpoint)
dump_results(folder_name, gateload_results,
sync_gateway_results)
except RequestException as re:
# Should not happen unless sg crashes
finished_successfully = False
log_info(re)
log_info(
"ERROR {}: sync_gateway not reachable. Dumping results to {}"
.format(re, folder_name))
dump_results(folder_name, gateload_results,
sync_gateway_results)
sg_is_running = False
dump_results(folder_name, gateload_results, sync_gateway_results)
log_info("Elapsed: {} minutes".format(
(time.time() - start_time) / 60.0))
time.sleep(sleep_time)
except RuntimeError as e:
log_info("Exception trying to log expvars: {}".format(e))
finished_successfully = False
return finished_successfully
from keywords.utils import log_info
if __name__ == "__main__":
usage = """
usage: python monitor_sync_gateway.py"
"""
try:
cluster_conf = os.environ["CLUSTER_CONFIG"]
except KeyError as ke:
log_info("Make sure CLUSTER_CONFIG is defined and pointing to the configuration you would like to provision")
raise KeyError("CLUSTER_CONFIG not defined. Unable to provision cluster.")
# Get sync_gateway ips from ansible inventory
sgs_host_vars = hosts_for_tag(cluster_conf, "sync_gateways")
sgs = [sgv["ansible_host"] for sgv in sgs_host_vars]
if len(sgs) == 0:
print("No sync_gateways to monitor in 'provisioning_config'")
sys.exit(1)
print("Monitoring sync_gateways: {}".format(sgs))
sgs_with_port = [sg + ":4985" for sg in sgs]
sgs_joined = ",".join(sgs_with_port)
expvars = [
"mem:memstats.Alloc",
"mem:memstats.Sys",
"mem:memstats.HeapAlloc",
"mem:memstats.HeapInuse",
import subprocess
import sys
from provisioning_config_parser import hosts_for_tag
from optparse import OptionParser
if __name__ == "__main__":
usage = """
usage: python monitor_sync_gateway.py"
"""
# Get sync_gateway ips from ansible inventory
sgs_host_vars = hosts_for_tag("sync_gateways")
sgs = [sgv["ansible_host"] for sgv in sgs_host_vars]
if len(sgs) == 0:
print("No sync_gateways to monitor in 'provisioning_config'")
sys.exit(1)
print("Monitoring sync_gateways: {}".format(sgs))
sgs_with_port = [sg + ":4985" for sg in sgs]
sgs_joined = ",".join(sgs_with_port)
expvars = [
"mem:memstats.Alloc",
"mem:memstats.Sys",
"mem:memstats.HeapAlloc",
"mem:memstats.HeapInuse",
"memstats.NumGC",
"memstats.PauseTotalNs",
"memstats.PauseNs",
from keywords.utils import log_info
if __name__ == "__main__":
usage = """
usage: python monitor_gateload.py"
"""
try:
cluster_conf = os.environ["CLUSTER_CONFIG"]
except KeyError as ke:
log_info("Make sure CLUSTER_CONFIG is defined and pointing to the configuration you would like to provision")
raise KeyError("CLUSTER_CONFIG not defined. Unable to provision cluster.")
# Get gateload ips from ansible inventory
lgs_host_vars = hosts_for_tag(cluster_conf, "load_generators")
lgs = [lg["ansible_host"] for lg in lgs_host_vars]
if len(lgs) == 0:
print("No gateloads to monitor in 'provisioning_config'")
sys.exit(1)
print("Monitoring gateloads: {}".format(lgs))
lgs_with_port = [lg + ":9876" for lg in lgs]
lgs_joined = ",".join(lgs_with_port)
expvars = [
"gateload.ops.PushToSubscriberInteractive.p95",
"gateload.ops.PushToSubscriberInteractive.p99",
"gateload.total_doc_pulled",
"gateload.total_doc_pushed",
import subprocess
import sys
from provisioning_config_parser import hosts_for_tag
if __name__ == "__main__":
usage = """
usage: python monitor_gateload.py"
"""
# Get gateload ips from ansible inventory
lgs_host_vars = hosts_for_tag("load_generators")
lgs = [lg["ansible_host"] for lg in lgs_host_vars]
if len(lgs) == 0:
print("No gateloads to monitor in 'provisioning_config'")
sys.exit(1)
print("Monitoring gateloads: {}".format(lgs))
lgs_with_port = [lg + ":9876" for lg in lgs]
lgs_joined = ",".join(lgs_with_port)
expvars = [
"gateload.ops.PushToSubscriberInteractive.p95",
"gateload.ops.PushToSubscriberInteractive.p99",
"gateload.total_doc_pulled",
"gateload.total_doc_pushed",
"gateload.user_active",
"gateload.user_awake",
"gateload.ops.AddUser.count"
]
