def multi_query(count, design_doc_name, view_name, params = None, bucket = "default", password = "", type_ = "view", batch_size = 100, hosts = None): if params is not None: params = urllib2.urllib.urlencode(params) pool = eventlet.GreenPool(batch_size) api = '%s/_design/%s/_%s/%s?%s' % (bucket, design_doc_name, type_, view_name, params) qtime = data = url = None args = dict(api=api, hosts=hosts) for qtime, data, url in pool.imap(send_query, [args for i in xrange(count)]): pass if cfg.SERIESLY_IP != '' and qtime is not None: # store the most recent query response time 'qtime' into seriesly seriesly = Seriesly(cfg.SERIESLY_IP, 3133) #TODO: do not hardcode fast...we should have per/testdbs db='fast' seriesly[db].append({'query_latency' : qtime}) # log to logs/celery-query.log try: rc = data.read()[0:200] except Exception: rc = "exception reading query response" logger.error('\n') logger.error('url: %s' % url) logger.error('latency: %s' % qtime) logger.error('data: %s' % rc)
def generate_index_file(storage_folder, test_file): db_event = Seriesly(cfg.SERIESLY_IP, 3133)['event'] all_event_docs = db_event.get_all() phases_info = {} for doc in all_event_docs.itervalues(): phases_info[int(doc.keys()[0])] = doc.values()[0] phases_info.keys().sort() num_phases = len(phases_info.keys()) run_id = phases_info[1]['desc'] run_id = run_id.replace(" ", "_") run_id = run_id.replace(",", "_") content = "" json_data = open(test_file) tests = json.load(json_data) for i in range(num_phases)[1:]: sub_folder = storage_folder + "phase" + str(i) + "/" content += "<a style=\"font-family:arial;color:black;font-size:20px;\" href=\"%s\">%s</a><p>" % ( "phase" + str(i), "phase" + str(i)) if str(i) in tests["phases"]: content += json.dumps( tests["phases"][str(i)], indent=10, sort_keys=True) + "<p>" files = [ f for f in os.listdir(sub_folder) if os.path.isfile(os.path.join(sub_folder, f)) ] for f in files: content += "<a href=\"%s\"> %s</a><p>" % ( "phase" + str(i) + "/" + f, f) html_path = storage_folder + "index.html" file1 = open(html_path, 'w') file1.write(index_html % content)
def main(): # parse database name from cli arguments db_name = parse_args() # initialize seriesly client db = Seriesly()[db_name] # get a set of all unique keys all_docs = db.get_all() all_keys = set(key for doc in all_docs.itervalues() for key in doc.iterkeys()) # plot all metrics to PNG images outdir = mkdtemp() for metric in all_keys: print metric if '/' not in metric: # views and xdcr stuff keys, values = get_metric(db, metric) plot_metric(metric, keys, values, outdir) try: subprocess.call(['convert', '{0}/*'.format(outdir), 'report.pdf']) print "PDF report was successfully generated!" except OSError: print "All images saved to: {0}".format(outdir)
def get_run_info(desc): db_event = Seriesly(cfg.SERIESLY_IP, 3133)['event'] all_event_docs = db_event.get_all() phases_info = {} for doc in all_event_docs.itervalues(): phases_info[int(doc.keys()[0])] = doc.values()[0] phases_info.keys().sort() run_info = '' #will take the first name/desc value; but we assume that their values are the same for all phases if desc == 'name': for phase in phases_info: if 'name' in phases_info[phase]: run_info = phases_info[phase]['name'] break if desc == 'build': for phase in phases_info: if 'desc' in phases_info[phase]: run_info = phases_info[phase]['desc'] break run_info = run_info.replace(" ", "_") run_info = run_info.replace(",", "_") return run_info
def __init__(self, in_host, out_host, database): self.url = 'http://{0}:8091/'.format(in_host) + \ 'pools/default/buckets/default/stats?zoom=minute' self.database = database self.seriesly = Seriesly(host=out_host) if database not in self.seriesly.list_dbs(): self.seriesly.create_db(database)
def __init__(self, test): self.seriesly = Seriesly(CBMONITOR_HOST) self.test_config = test.test_config self.metric_title = test.test_config.test_case.metric_title self.cluster_spec = test.cluster_spec self.cluster_names = test.cbagent.clusters.keys() self.build = test.build self.master_node = test.master_node
def __init__(self, test): self.seriesly = Seriesly(SERIESLY['host']) self.test_config = test.test_config self.metric_title = test.test_config.test_case.metric_title self.cluster_spec = test.cluster_spec self.cluster_names = test.cbagent.clusters.keys() self.build = test.build self.master_node = test.master_node
def start_samplers(self): logger.info('Creating seriesly dbs') seriesly = Seriesly( host='{}'.format(self.test_config.gateload_settings.seriesly_host)) for i, _ in enumerate(self.remote.gateways, start=1): seriesly.create_db('gateway_{}'.format(i)) seriesly.create_db('gateload_{}'.format(i)) self.remote.start_sampling()
def plot_all_phases(cluster_name, buckets): db_event = Seriesly(cfg.SERIESLY_IP, 3133)['event'] # Get system test phase info and plot phase by phase all_event_docs = db_event.get_all() phases_info = {} for doc in all_event_docs.values(): phases_info[int(list(doc.keys())[0])] = list(doc.values())[0] list(phases_info.keys()).sort() num_phases = len(list(phases_info.keys())) run_id = store_report.get_run_info('name') if not os.path.exists("%s" % run_id): os.makedirs("%s" % run_id) else: shutil.rmtree("%s" % run_id) os.makedirs("%s" % run_id) for i in list(phases_info.keys()): start_time = phases_info[i][[ name for name in list(phases_info[i].keys()) if (name != 'name' and name != 'desc') ][0]] start_time = int(start_time[:10]) end_time = 0 if i == list(phases_info.keys())[-1]: end_time = str(time.time()) end_time = int(end_time[:10]) else: end_time = phases_info[i + 1][[ name for name in list(phases_info[i + 1].keys()) if (name != 'name' and name != 'desc') ][0]] end_time = int(end_time[:10]) start_time_snapshot = datetime.datetime.fromtimestamp( start_time).strftime('%m/%d/%Y %H:%M') end_time_snapshot = datetime.datetime.fromtimestamp(end_time).strftime( '%m/%d/%Y %H:%M') snapshot_name = "phase-%d-%s" % (i, [ name for name in list(phases_info[i].keys()) if (name != 'name' and name != 'desc') ][0]) plot_use_cbmonitor(snapshot_name, cluster_name, start_time_snapshot, end_time_snapshot) prepare_folder_report(run_id, i) store_90th_avg_value(buckets, start_time, end_time, run_id, i) storage_folder = os.getcwd() + "/" + run_id + "/" print("data stored in %s" % (storage_folder)) return storage_folder
def add_phase_to_db(phase, phase_key, name, desc): if cfg.SERIESLY_IP != '': seriesly = Seriesly(cfg.SERIESLY_IP, 3133) seriesly.event.append({ str(phase_key): { str(phase['name']): str(time.time()), 'run_id': name + '-' + desc } })
def __init__(self, host, dbslow, dbfast): self.slow = {} self.fast = {} self.dbslow = dbslow self.dbfast = dbfast self.seriesly = Seriesly(host=host) try: dbs = self.seriesly.list_dbs() except seriesly.exceptions.ConnectionError, e: logging.error("unable to connect to seriesly server: %s" % e) return
def __init__(self): self.db = Seriesly() self.fig = figure() self.fig.set_size_inches(4.66, 2.625) self.urls = list() self.images = list() self.eventlet_pool = GreenPool() self.mp_pool = Pool(cpu_count())
def multi_query(count, design_doc_name, view_name, params=None, bucket="default", password="", type_="view", batch_size=100, hosts=None): if params is not None: params = urllib2.urllib.urlencode(params) pool = eventlet.GreenPool(batch_size) api = '%s/_design/%s/_%s/%s?%s' % (bucket, design_doc_name, type_, view_name, params) qtime = data = url = None args = dict(api=api, hosts=hosts) for qtime, data, url in pool.imap(send_query, [args for i in xrange(count)]): pass if cfg.SERIESLY_IP != '' and qtime is not None: # store the most recent query response time 'qtime' into seriesly seriesly = Seriesly(cfg.SERIESLY_IP, 3133) db = None if 'fast' in seriesly.list_dbs(): db = 'fast' else: bucketStatus = app.workload_manager.BucketStatus.from_cache( bucket) or app.workload_manager.BucketStatus(bucket) db = bucketStatus.latency_db if db not in seriesly.list_dbs(): seriesly.create_db(db) if db is not None: seriesly[db].append({'query_latency': qtime}) # log to logs/celery-query.log try: rc = data.read()[0:200] except Exception: rc = "exception reading query response" logger.error('\n') logger.error('url: %s' % url) logger.error('latency: %s' % qtime) logger.error('data: %s' % rc)
def __init__(self, host): self.seriesly = Seriesly(host)
return val metrics = [ "secondary_scanlatency20M_fdb_nyx", "secondary_scanlatency20M_multiple_fdb_nyx", "secondary_scanlatency_rebalance20M_fdb_nyx", "secondary_scanlatency_stalefalse_20M_fdb_nyx", "secondary_doc_indexing_latency_20M_moi_80th_nyx_query", "secondary_scanlatency20M_moi_nyx", "secondary_scanlatency20M_multiple_moi_nyx", "secondary_scanlatency_rebalance20M_moi_nyx", "secondary_scanlatency_stalefalse_20M_100Kops_moi_nyx", ] s = Seriesly(host='cbmonitor.sc.couchbase.com') b = Bucket("couchbase://cbmonitor.sc.couchbase.com/benchmarks", password="******") for metric in metrics: print "********* Metric: " + metric q = N1QLQuery( 'SELECT id,snapshots FROM benchmarks WHERE metric = "{}";'.format( metric)) for row in b.n1ql_query(q): doc_id = row['id'] snapshot = row['snapshots'][0] if len(row['snapshots']) > 1:
def report_kv_latency(bucket="default"): if cfg.SERIESLY_IP == '': # seriesly not configured return rabbitHelper = report_kv_latency.rabbitHelper clusterStatus = CacheHelper.clusterstatus(cfg.CB_CLUSTER_TAG+"_status") or\ ClusterStatus() host = clusterStatus.get_random_host() if host is None: return ip, port = host.split(':') workloads = CacheHelper.workloads() for workload in workloads: if workload.active and workload.bucket == bucket: # read workload params bucket = str(workload.bucket) password = str(workload.password) # read template from active workload template = Template.from_cache(str(workload.template)) template = template.__dict__ client.decodeMajgicStrings(template) # setup key/val to use for timing key = _random_string(12) value = json.dumps(template['kv']) get_key = key # for get op, try to pull from consume_queue # so that we can calc impact of dgm consume_queue = workload.consume_queue if consume_queue is not None: keys = rabbitHelper.getJsonMsg(str(consume_queue), requeue=True) if len(keys) > 0: get_key = str(keys['start']) # collect op latency set_latency = client.mc_op_latency('set', key, value, ip, port, bucket, password) get_latency = client.mc_op_latency('get', get_key, value, ip, port, bucket, password) delete_latency = client.mc_op_latency('delete', key, value, ip, port, bucket, password) # report to seriessly seriesly = Seriesly(cfg.SERIESLY_IP, 3133) db = None if 'fast' in seriesly.list_dbs(): db = 'fast' else: bucketStatus = BucketStatus.from_cache(bucket) or BucketStatus( bucket) db = bucketStatus.latency_db if db not in seriesly.list_dbs(): seriesly.create_db(db) if db is not None: seriesly[db].append({ 'set_latency': set_latency, 'get_latency': get_latency, 'delete_latency': delete_latency })
def store_90th_avg_value(buckets, start_time, end_time, run_id, i): ips = get_cluster_ips() ns_server_stats = None atop_stats = None latency_stats = [ 'set_latency', 'get_latency', 'delete_latency', 'query_latency' ] dict_90th = {} dict_avg = {} dict_90th['ns_server'] = {} dict_avg['ns_server'] = {} time.sleep(2) connection = Seriesly(cfg.SERIESLY_IP, 3133) for bucket in buckets: toolbar_width = 41 sys.stdout.write("[%s] indicator\n" % ("*" * toolbar_width)) sys.stdout.flush() dict_90th['ns_server'][bucket] = {} dict_avg['ns_server'][bucket] = {} #empty ip string appended with bucket name is the ns_server db name for entire cluster cluster_ips = ips cluster_ips.append('') for ip in cluster_ips: ns_server_db = "ns_serverdefault" + bucket + ip if ":" in ns_server_db: ns_server_db = ns_server_db[0:ns_server_db.find(":")] if ip == '': ip = 'cluster' dict_90th['ns_server'][bucket][ip] = {} dict_avg['ns_server'][bucket][ip] = {} if ns_server_db not in connection.list_dbs(): print("db %s was not found" % (ns_server_db)) continue db = connection[ns_server_db] if ns_server_stats is None: ns_server_stats = list(db.get_all().values())[0].keys() print("Store ns server stats for bucket %s on %s" % (bucket, ip)) sys.stdout.write("[") num = 1 for metric in ns_server_stats: dict_90th['ns_server'][bucket][ip][metric] = store_90th_value( db, metric, start_time, end_time) dict_avg['ns_server'][bucket][ip][metric] = store_avg_value( db, metric, start_time, end_time) if num % (len(ns_server_stats) // toolbar_width) == 0: sys.stdout.write("=") sys.stdout.flush() time.sleep(0.5) num += 1 sys.stdout.write("]\n") dict_90th['atop'] = {} dict_avg['atop'] = {} for ip in ips: atop_db = "atopdefault" + ip dict_90th['atop'][ip] = {} dict_avg['atop'][ip] = {} if ":" in atop_db: atop_db = atop_db[0:atop_db.find(":")] if atop_db not in connection.list_dbs(): print("db %s was not found" % (atop_db)) continue db = connection[atop_db] if atop_stats is None: atop_stats = list(db.get_all().values())[0].keys() print("Store atop stats for node %s" % (ip)) for metric in atop_stats: dict_90th['atop'][ip][metric] = store_90th_value( db, metric, start_time, end_time) dict_avg['atop'][ip][metric] = store_avg_value( db, metric, start_time, end_time) dict_90th['latency'] = {} dict_avg['latency'] = {} for bucket in buckets: dict_90th['latency'][bucket] = {} dict_avg['latency'][bucket] = {} latency_db = "%slatency" % bucket if latency_db not in connection.list_dbs(): print("db %s was not found" % (latency_db)) continue db = connection[latency_db] print("Store latency stats for bucket %s" % (bucket)) for metric in latency_stats: dict_90th['latency'][bucket][metric] = store_90th_value( db, metric, start_time, end_time) dict_avg['latency'][bucket][metric] = store_avg_value( db, metric, start_time, end_time) os.system('rm -f %s/phase%d/*.txt' % (run_id, i)) json.dump(dict_90th, open("%s/phase%d/90percentile.txt" % (run_id, i), 'w')) json.dump(dict_avg, open("%s/phase%d/average.txt" % (run_id, i), 'w')) del dict_90th del dict_avg
def __init__(self, *args, **kwargs): super(SgwMetricHelper, self).__init__(*args, **kwargs) self.seriesly = Seriesly( self.test_config.gateload_settings.seriesly_host)
def main(): seriesly = Seriesly() all_dbs = seriesly.list_dbs() for i, db in enumerate(all_dbs, start=1): logger.info("{}/{}: {}".format(i, len(all_dbs), db.strip())) seriesly[db.strip()].compact()
for proc in kill_procs: os.system("ps aux | grep %s | awk '{print $2}' | xargs kill" % proc) # delete queues (note using --purge will remove cc_queues) queues = CacheHelper.task_queues() + CacheHelper.miss_queues() # when --purge set delete cc_queue's as well # as seriesly db if "--purge" in sys.argv: queues = set(CacheHelper.queues()) # cleaning up seriesly database (fast and slow created by cbtop) if cfg.SERIESLY_IP != '': from seriesly import Seriesly seriesly = Seriesly(cfg.SERIESLY_IP, 3133) dbs = seriesly.list_dbs() for db in dbs: seriesly.drop_db(db) seriesly.create_db('event') for q_ in queues: try: RabbitHelper().delete(q_) print("Cleanup Queue: %s" % q_) except Exception as ex: pass # clean up cache CacheHelper.cacheClean()
def init_client(): world.client = Seriesly(host=world.config.get('database', 'host'), port=world.config.get('database', 'port'))