def log_expvars(cluster_config, folder_name): """ usage: log_expvars.py" """ # Get gateload ips from ansible inventory lgs_host_vars = hosts_for_tag(cluster_config, "load_generators") lgs = [lg["ansible_host"] for lg in lgs_host_vars] lgs_expvar_endpoints = [lg + ":9876/debug/vars" for lg in lgs] log_info("Monitoring gateloads until they finish: {}".format("\n".join(lgs_expvar_endpoints))) # Get sync_gateway ips from ansible inventory sgs_host_vars = hosts_for_tag(cluster_config, "sync_gateways") sgs = [sgv["ansible_host"] for sgv in sgs_host_vars] sgs_expvar_endpoints = [sg + ":4985/_expvar" for sg in sgs] log_info("Monitoring sync_gateways: {}".format("\n".join(sgs_expvar_endpoints))) # Verify that sync gateway expvar endpoints are reachable wait_for_endpoints_alive_or_raise(sgs_expvar_endpoints) # Wait until the gateload expvar endpoints are up, or raise an exception and abort wait_for_endpoints_alive_or_raise(lgs_expvar_endpoints) start_time = time.time() gateload_results = OrderedDict() sync_gateway_results = OrderedDict() gateload_is_running = True while gateload_is_running: # Caputure expvars for gateloads for endpoint in lgs_expvar_endpoints: try: write_expvars(gateload_results, endpoint) except ConnectionError as he: # connection to gateload expvars has been closed log_info("Gateload {} no longer reachable. Writing expvars to {}".format(endpoint, folder_name)) dump_results(folder_name, gateload_results, sync_gateway_results) gateload_is_running = False # Capture expvars for sync_gateways for endpoint in sgs_expvar_endpoints: try: write_expvars(sync_gateway_results, endpoint) except ConnectionError as he: # Should not happen unless sg crashes log_info(he) log_info("ERROR: sync_gateway not reachable. Dumping results to {}".format(folder_name)) dump_results(folder_name, gateload_results, sync_gateway_results) log_info("Elapsed: {} minutes".format((time.time() - start_time) / 60.0)) time.sleep(30)
def log_expvars(folder_name): usage = """ usage: log_expvars.py" """ # Get gateload ips from ansible inventory lgs_host_vars = hosts_for_tag("load_generators") lgs = [lg["ansible_host"] for lg in lgs_host_vars] print("Monitoring gateloads: {}".format(lgs)) lgs_expvar_endpoints = [lg + ":9876/debug/vars" for lg in lgs] # Get sync_gateway ips from ansible inventory sgs_host_vars = hosts_for_tag("sync_gateways") sgs = [sgv["ansible_host"] for sgv in sgs_host_vars] print("Monitoring sync_gateways: {}".format(sgs)) sgs_expvar_endpoints = [sg + ":4985/_expvar" for sg in sgs] start_time = time.time() gateload_results = OrderedDict() sync_gateway_results = OrderedDict() gateload_is_running = True while gateload_is_running: # Caputure expvars for gateloads for endpoint in lgs_expvar_endpoints: try: write_expvars(gateload_results, endpoint) except ConnectionError as he: # connection to gateload expvars has been closed print(he) print("Gateload no longer reachable. Writing expvars ...") dump_results(folder_name, gateload_results, sync_gateway_results) gateload_is_running = False # Capture expvars for sync_gateways for endpoint in sgs_expvar_endpoints: try: write_expvars(sync_gateway_results, endpoint) except ConnectionError as he: # Should not happen unless sg crashes print(he) print("ERROR: sync_gateway not reachable. Dumping results") dump_results(folder_name, gateload_results, sync_gateway_results) print("Elapsed: {} minutes".format((time.time() - start_time) / 60.0)) time.sleep(30)
if __name__ == "__main__": usage = """ usage: python monitor_gateload.py" """ try: cluster_conf = os.environ["CLUSTER_CONFIG"] except KeyError as ke: log_info( "Make sure CLUSTER_CONFIG is defined and pointing to the configuration you would like to provision" ) raise KeyError( "CLUSTER_CONFIG not defined. Unable to provision cluster.") # Get gateload ips from ansible inventory lgs_host_vars = hosts_for_tag(cluster_conf, "load_generators") lgs = [lg["ansible_host"] for lg in lgs_host_vars] if len(lgs) == 0: print("No gateloads to monitor in 'provisioning_config'") sys.exit(1) print("Monitoring gateloads: {}".format(lgs)) lgs_with_port = [lg + ":9876" for lg in lgs] lgs_joined = ",".join(lgs_with_port) expvars = [ "gateload.ops.PushToSubscriberInteractive.p95", "gateload.ops.PushToSubscriberInteractive.p99", "gateload.total_doc_pulled", "gateload.total_doc_pushed", "gateload.user_active", "gateload.user_awake",
def log_expvars(cluster_config, folder_name, sleep_time=30): """ usage: log_expvars.py" """ finished_successfully = True try: # Get gateload ips from ansible inventory lgs_host_vars = hosts_for_tag(cluster_config, "load_generators") lgs = [lg["ansible_host"] for lg in lgs_host_vars] lgs_expvar_endpoints = [lg + ":9876/debug/vars" for lg in lgs] log_info("Monitoring gateloads until they finish: {}".format( "\n".join(lgs_expvar_endpoints))) # Get sync_gateway ips from ansible inventory sgs_host_vars = hosts_for_tag(cluster_config, "sync_gateways") sgs = [sgv["ansible_host"] for sgv in sgs_host_vars] sgs_expvar_endpoints = [sg + ":4985/_expvar" for sg in sgs] log_info("Monitoring sync_gateways: {}".format( "\n".join(sgs_expvar_endpoints))) # Verify that sync gateway expvar endpoints are reachable wait_for_endpoints_alive_or_raise(sgs_expvar_endpoints) # Wait until the gateload expvar endpoints are up, or raise an exception and abort wait_for_endpoints_alive_or_raise(lgs_expvar_endpoints) start_time = time.time() gateload_results = OrderedDict() sync_gateway_results = OrderedDict() gateload_is_running = True sg_is_running = True while gateload_is_running and sg_is_running: # Capture expvars for gateloads for endpoint in lgs_expvar_endpoints: try: log_info("Collecting gateload expvars {}".format(endpoint)) write_expvars(gateload_results, endpoint) dump_results(folder_name, gateload_results, sync_gateway_results) except RequestException as re: # connection to gateload expvars has been closed log_info(re) log_info( "Error: {}. Gateload {} no longer reachable. Writing expvars to {}" .format(re, endpoint, folder_name)) dump_results(folder_name, gateload_results, sync_gateway_results) gateload_is_running = False # Capture expvars for sync_gateways for endpoint in sgs_expvar_endpoints: try: log_info("Collecting sg expvars {}".format(endpoint)) write_expvars(sync_gateway_results, endpoint) dump_results(folder_name, gateload_results, sync_gateway_results) except RequestException as re: # Should not happen unless sg crashes finished_successfully = False log_info(re) log_info( "ERROR {}: sync_gateway not reachable. Dumping results to {}" .format(re, folder_name)) dump_results(folder_name, gateload_results, sync_gateway_results) sg_is_running = False dump_results(folder_name, gateload_results, sync_gateway_results) log_info("Elapsed: {} minutes".format( (time.time() - start_time) / 60.0)) time.sleep(sleep_time) except RuntimeError as e: log_info("Exception trying to log expvars: {}".format(e)) finished_successfully = False return finished_successfully
from keywords.utils import log_info if __name__ == "__main__": usage = """ usage: python monitor_sync_gateway.py" """ try: cluster_conf = os.environ["CLUSTER_CONFIG"] except KeyError as ke: log_info("Make sure CLUSTER_CONFIG is defined and pointing to the configuration you would like to provision") raise KeyError("CLUSTER_CONFIG not defined. Unable to provision cluster.") # Get sync_gateway ips from ansible inventory sgs_host_vars = hosts_for_tag(cluster_conf, "sync_gateways") sgs = [sgv["ansible_host"] for sgv in sgs_host_vars] if len(sgs) == 0: print("No sync_gateways to monitor in 'provisioning_config'") sys.exit(1) print("Monitoring sync_gateways: {}".format(sgs)) sgs_with_port = [sg + ":4985" for sg in sgs] sgs_joined = ",".join(sgs_with_port) expvars = [ "mem:memstats.Alloc", "mem:memstats.Sys", "mem:memstats.HeapAlloc", "mem:memstats.HeapInuse",
import subprocess import sys from provisioning_config_parser import hosts_for_tag from optparse import OptionParser if __name__ == "__main__": usage = """ usage: python monitor_sync_gateway.py" """ # Get sync_gateway ips from ansible inventory sgs_host_vars = hosts_for_tag("sync_gateways") sgs = [sgv["ansible_host"] for sgv in sgs_host_vars] if len(sgs) == 0: print("No sync_gateways to monitor in 'provisioning_config'") sys.exit(1) print("Monitoring sync_gateways: {}".format(sgs)) sgs_with_port = [sg + ":4985" for sg in sgs] sgs_joined = ",".join(sgs_with_port) expvars = [ "mem:memstats.Alloc", "mem:memstats.Sys", "mem:memstats.HeapAlloc", "mem:memstats.HeapInuse", "memstats.NumGC", "memstats.PauseTotalNs", "memstats.PauseNs",
from keywords.utils import log_info if __name__ == "__main__": usage = """ usage: python monitor_gateload.py" """ try: cluster_conf = os.environ["CLUSTER_CONFIG"] except KeyError as ke: log_info("Make sure CLUSTER_CONFIG is defined and pointing to the configuration you would like to provision") raise KeyError("CLUSTER_CONFIG not defined. Unable to provision cluster.") # Get gateload ips from ansible inventory lgs_host_vars = hosts_for_tag(cluster_conf, "load_generators") lgs = [lg["ansible_host"] for lg in lgs_host_vars] if len(lgs) == 0: print("No gateloads to monitor in 'provisioning_config'") sys.exit(1) print("Monitoring gateloads: {}".format(lgs)) lgs_with_port = [lg + ":9876" for lg in lgs] lgs_joined = ",".join(lgs_with_port) expvars = [ "gateload.ops.PushToSubscriberInteractive.p95", "gateload.ops.PushToSubscriberInteractive.p99", "gateload.total_doc_pulled", "gateload.total_doc_pushed",
import subprocess import sys from provisioning_config_parser import hosts_for_tag if __name__ == "__main__": usage = """ usage: python monitor_gateload.py" """ # Get gateload ips from ansible inventory lgs_host_vars = hosts_for_tag("load_generators") lgs = [lg["ansible_host"] for lg in lgs_host_vars] if len(lgs) == 0: print("No gateloads to monitor in 'provisioning_config'") sys.exit(1) print("Monitoring gateloads: {}".format(lgs)) lgs_with_port = [lg + ":9876" for lg in lgs] lgs_joined = ",".join(lgs_with_port) expvars = [ "gateload.ops.PushToSubscriberInteractive.p95", "gateload.ops.PushToSubscriberInteractive.p99", "gateload.total_doc_pulled", "gateload.total_doc_pushed", "gateload.user_active", "gateload.user_awake", "gateload.ops.AddUser.count" ]